diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index a38ffc1f..8d5cddd7 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "webcmd", - "version": "0.6.1", + "version": "0.6.2", "description": "Turn websites, browser sessions, desktop apps, and local tools into deterministic CLI surfaces for humans and AI agents.", "author": { "name": "AgentRHQ", diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 9207a49b..f255894f 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "webcmd", - "version": "0.6.1", + "version": "0.6.2", "description": "Turn websites, browser sessions, desktop apps, and local tools into deterministic CLI surfaces for humans and AI agents.", "author": { "name": "AgentRHQ", diff --git a/cli-manifest.json b/cli-manifest.json index cc96cc2c..c3a91a9b 100644 --- a/cli-manifest.json +++ b/cli-manifest.json @@ -1,95 +1,43 @@ [ { "site": "web", - "name": "fetch-browser", - "description": "Fetch any web page and export as Markdown", + "name": "fetch", + "description": "Fetch a URL with local HTTP clients", "access": "read", - "strategy": "cookie", - "browser": true, + "strategy": "public", + "browser": false, "args": [ { "name": "url", - "type": "str", + "type": "string", "required": true, - "help": "Any web page URL" + "help": "HTTP or HTTPS URL to fetch" }, { - "name": "output", - "type": "str", - "default": "./web-articles", - "required": false, - "help": "Output directory" - }, - { - "name": "download-images", - "type": "boolean", - "default": true, - "required": false, - "help": "Download images locally" - }, - { - "name": "wait", + "name": "timeout", "type": "int", - "default": 3, - "required": false, - "help": "Seconds to wait after page load" - }, - { - "name": "wait-for", - "type": "str", + "default": 30, "required": false, - "valueRequired": true, - "help": "CSS selector to wait for in the main document or same-origin iframes" + "help": "Total fetch budget in seconds" }, { - "name": "wait-until", - "type": "str", - "default": "domstable", - "required": false, - "help": "Readiness policy after navigation: domstable or networkidle", - "choices": [ - "domstable", - "networkidle" - ] - }, - { - "name": "frames", - "type": "str", - "default": "same-origin", - "required": false, - "help": "Iframe handling mode: relevant same-origin, all-same-origin, or none", - "choices": [ - "same-origin", - "all-same-origin", - "none" - ] - }, - { - "name": "diagnose", - "type": "boolean", - "default": false, + "name": "max-chars", + "type": "int", + "default": 50000, "required": false, - "help": "Print render diagnostics (frames, empty containers, XHR/API-like requests) to stderr" + "help": "Maximum extracted characters; 0 disables truncation" }, { - "name": "stdout", + "name": "allow-private", "type": "boolean", "default": false, "required": false, - "help": "Print markdown to stdout instead of saving to a file" + "help": "Allow private and loopback destinations" } ], - "columns": [ - "title", - "author", - "publish_time", - "status", - "size", - "saved" - ], + "defaultFormat": "md", "type": "js", - "modulePath": "web/fetch-browser.js", - "sourceFile": "web/fetch-browser.js", - "navigateBefore": false + "clientOwned": true, + "packageExport": "./fetch/command" } ] diff --git a/clis/web/README.md b/clis/web/README.md deleted file mode 100644 index a4f69f9b..00000000 --- a/clis/web/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# webcmd web - -Bundled with `webcmd` by default. - -## Commands - -| Command | Description | -| --- | --- | -| `webcmd web fetch` | Fetch a URL locally without launching a browser | -| `webcmd web fetch-browser` | Fetch any web page and export as Markdown | diff --git a/clis/web/fetch-browser.js b/clis/web/fetch-browser.js deleted file mode 100644 index c620a37e..00000000 --- a/clis/web/fetch-browser.js +++ /dev/null @@ -1,491 +0,0 @@ -/** - * Generic web page reader — fetch any URL and export as Markdown. - * - * Uses browser-side DOM heuristics to extract the main content: - * 1.
element - * 2. [role="main"] element - * 3.
element - * 4. Largest text-dense block as fallback - * - * Pipes through the shared article-download pipeline (Turndown + image download). - * - * Usage: - * webcmd web fetch-browser --url "https://www.anthropic.com/research/..." --output ./articles - * webcmd web fetch-browser --url "https://..." --download-images false - */ -import { cli, Strategy } from '@agentrhq/webcmd/registry'; -import { downloadArticle } from '@agentrhq/webcmd/download/article-download'; - -const NETWORK_IDLE_QUIET_MS = 1000; -const NETWORK_IDLE_POLL_MS = 500; -const MIN_NON_STRUCTURAL_IFRAME_TEXT_CHARS = 50; - -function sleep(ms) { - return new Promise(resolve => setTimeout(resolve, ms)); -} - -function boolish(value) { - if (value === true) return true; - if (typeof value === 'string') return ['1', 'true', 'yes', 'on'].includes(value.toLowerCase()); - return false; -} - -function normalizeFrameMode(value) { - const mode = String(value || 'same-origin').toLowerCase(); - if (['same-origin', 'all-same-origin', 'none'].includes(mode)) return mode; - return 'same-origin'; -} - -function normalizeWaitUntil(value) { - const waitUntil = String(value || 'domstable').toLowerCase(); - if (['domstable', 'networkidle'].includes(waitUntil)) return waitUntil; - return 'domstable'; -} - -function normalizeNetworkEntry(entry) { - const preview = typeof entry?.responsePreview === 'string' ? entry.responsePreview : ''; - return { - method: typeof entry?.method === 'string' ? entry.method : 'GET', - url: typeof entry?.url === 'string' ? entry.url : '', - status: typeof entry?.responseStatus === 'number' ? entry.responseStatus : 0, - contentType: typeof entry?.responseContentType === 'string' ? entry.responseContentType : '', - size: typeof entry?.responseBodyFullSize === 'number' ? entry.responseBodyFullSize : preview.length, - bodyTruncated: entry?.responseBodyTruncated === true, - }; -} - -function isInterestingNetworkEntry(entry) { - const ct = (entry.contentType || '').toLowerCase(); - const url = entry.url || ''; - const method = (entry.method || 'GET').toUpperCase(); - const staticAsset = /\.(js|css|png|jpg|jpeg|gif|svg|woff|woff2|ico|map)(\?|$)/i.test(url); - const noisy = /analytics|tracking|telemetry|beacon|pixel|gtag|fbevents/i.test(url); - const apiLikeUrl = /\/(api|ajax|graphql|rest|service|handler)(\/|[?._-]|$)|\.(ashx|aspx|asmx|php)(\?|$)/i.test(url); - const dataLikeContent = ct.includes('json') - || ct.includes('xml') - || ct.includes('text/plain') - || ct.includes('javascript') - || (apiLikeUrl && ct.includes('text/html')); - return ( - !staticAsset - && !noisy - && (dataLikeContent || apiLikeUrl || method !== 'GET') - ); -} - -async function drainNetworkCapture(page, sink) { - if (!page.readNetworkCapture) return []; - const raw = await page.readNetworkCapture().catch(() => []); - const entries = Array.isArray(raw) ? raw.map(normalizeNetworkEntry).filter(entry => entry.url) : []; - sink.push(...entries); - return entries; -} - -async function maybeStartNetworkCapture(page) { - if (!page.startNetworkCapture) return false; - try { - return await page.startNetworkCapture(''); - } catch { - return false; - } -} - -async function waitForNetworkIdle(page, maxSeconds, sink) { - const timeoutMs = Math.max(1, Number(maxSeconds) || 1) * 1000; - const deadline = Date.now() + timeoutMs; - let quietSince = Date.now(); - while (Date.now() < deadline) { - const entries = await drainNetworkCapture(page, sink); - if (entries.length > 0) quietSince = Date.now(); - if (Date.now() - quietSince >= NETWORK_IDLE_QUIET_MS) return { ok: true }; - await sleep(NETWORK_IDLE_POLL_MS); - } - return { ok: false, timedOut: true }; -} - -function buildWaitForSelectorAcrossFramesJs(selector, timeoutMs) { - return ` - (async () => { - const selector = ${JSON.stringify(selector)}; - const timeoutAt = Date.now() + ${Number(timeoutMs) || 10000}; - const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)); - const sameOriginFrameDocs = () => Array.from(document.querySelectorAll('iframe')).map((frame) => { - try { - const href = new URL(frame.getAttribute('src') || frame.src || '', window.location.href).href; - if (new URL(href).origin !== window.location.origin) return null; - return { href, doc: frame.contentDocument }; - } catch { - return null; - } - }).filter(Boolean); - const findMatch = () => { - try { - if (document.querySelector(selector)) return { ok: true, scope: 'main', url: window.location.href }; - } catch (err) { - return { ok: false, invalidSelector: true, error: String(err && err.message || err) }; - } - for (const frame of sameOriginFrameDocs()) { - try { - if (frame.doc?.querySelector(selector)) return { ok: true, scope: 'iframe', url: frame.href }; - } catch {} - } - return { ok: false }; - }; - while (Date.now() < timeoutAt) { - const found = findMatch(); - if (found.ok || found.invalidSelector) return found; - await sleep(100); - } - return { ok: false, timedOut: true, selector }; - })() - `; -} - -function buildRenderAwareExtractorJs(options) { - return ` - (() => { - const frameMode = ${JSON.stringify(options.frames)}; - const minNonStructuralIframeTextChars = ${MIN_NON_STRUCTURAL_IFRAME_TEXT_CHARS}; - const result = { - title: '', - author: '', - publishTime: '', - contentHtml: '', - imageUrls: [], - diagnostics: { - url: window.location.href, - frames: [], - emptyContainers: [], - includedFrameCount: 0 - } - }; - - const absolutize = (value, base) => { - if (!value || value.startsWith('data:') || value.startsWith('javascript:') || value.startsWith('#')) return value || ''; - try { return new URL(value, base).href; } catch { return value; } - }; - const absolutizeTree = (root, base) => { - root.querySelectorAll?.('[href]').forEach(el => el.setAttribute('href', absolutize(el.getAttribute('href'), base))); - root.querySelectorAll?.('[src]').forEach(el => el.setAttribute('src', absolutize(el.getAttribute('src'), base))); - root.querySelectorAll?.('[poster]').forEach(el => el.setAttribute('poster', absolutize(el.getAttribute('poster'), base))); - root.querySelectorAll?.('[action]').forEach(el => el.setAttribute('action', absolutize(el.getAttribute('action'), base))); - }; - const textLen = (node) => (node?.textContent || '').replace(/\\s+/g, ' ').trim().length; - const describeFrame = (frame, index) => { - const rawSrc = frame.getAttribute('src') || frame.src || ''; - let href = ''; - try { href = new URL(rawSrc, window.location.href).href; } catch { href = rawSrc; } - let sameOrigin = false; - try { sameOrigin = href ? new URL(href).origin === window.location.origin : false; } catch {} - let accessible = false; - let title = frame.getAttribute('title') || frame.getAttribute('name') || frame.id || ''; - let length = 0; - try { - accessible = !!frame.contentDocument; - title = title || frame.contentDocument?.title || ''; - length = textLen(frame.contentDocument?.body); - } catch {} - return { index, src: href, title, sameOrigin, accessible, textLength: length }; - }; - const collectEmptyContainers = (root, scope, baseUrl) => { - const likely = 'table, tbody, ul[id], ol[id], div[id], section[id], [class*="grid"], [class*="data"], [class*="list"], [id*="grid"], [id*="data"], [id*="list"]'; - root.querySelectorAll?.(likely).forEach((el) => { - if (scope === 'main' && el.closest?.('[data-webcmd-iframe-source]')) return; - const id = el.getAttribute('id') || ''; - const cls = el.getAttribute('class') || ''; - const name = [id, cls].join(' ').toLowerCase(); - if (!/(grid|data|list|table|content|result)/.test(name) && !['TABLE', 'TBODY', 'UL', 'OL'].includes(el.nodeName)) return; - if (textLen(el) > 20) return; - result.diagnostics.emptyContainers.push({ - scope, - url: baseUrl, - tag: el.tagName.toLowerCase(), - id, - className: cls, - }); - }); - }; - const hasDataContainerSignal = (root) => { - const likely = 'table, tbody, ul[id], ol[id], [id*="grid"], [id*="data"], [id*="list"], [id*="content"], [id*="result"], [class*="grid"], [class*="data"], [class*="list"], [class*="content"], [class*="result"]'; - return !!root.querySelector?.(likely); - }; - const shouldIncludeExternalFrame = (frameBody) => { - // Outside-content iframes are less trusted than placeholders inside - // contentEl. Long plain text is the fallback for simple same-origin - // frames that lack article/table/list structure. - if (textLen(frameBody) >= minNonStructuralIframeTextChars) return true; - if (frameBody.querySelector?.('article, main, [role="main"], table, tbody, ul li, ol li')) return true; - return hasDataContainerSignal(frameBody); - }; - const buildFrameSection = (frameBody, desc, fallbackLabel) => { - absolutizeTree(frameBody, desc.src || window.location.href); - collectEmptyContainers(frameBody, 'iframe', desc.src); - const section = document.createElement('section'); - section.setAttribute('data-webcmd-iframe-source', desc.src); - const heading = document.createElement('h2'); - heading.textContent = 'localized text iframe: ' + (desc.src || fallbackLabel); - section.appendChild(heading); - Array.from(frameBody.childNodes).forEach(node => section.appendChild(node)); - return section; - }; - - const ogTitle = document.querySelector('meta[property="og:title"]'); - if (ogTitle) result.title = ogTitle.getAttribute('content')?.trim() || ''; - if (!result.title) result.title = document.title?.trim() || ''; - if (!result.title) result.title = document.querySelector('h1')?.textContent?.trim() || 'untitled'; - result.title = result.title.replace(/\\s*[|\\-–—]\\s*[^|\\-–—]{1,30}$/, '').trim(); - - const authorMeta = document.querySelector('meta[name="author"], meta[property="article:author"], meta[name="twitter:creator"]'); - result.author = authorMeta?.getAttribute('content')?.trim() || ''; - - const timeMeta = document.querySelector('meta[property="article:published_time"], meta[name="date"], meta[name="publishdate"], time[datetime]'); - if (timeMeta) { - result.publishTime = timeMeta.getAttribute('content') - || timeMeta.getAttribute('datetime') - || timeMeta.textContent?.trim() - || ''; - } - - let contentEl = null; - const articles = document.querySelectorAll('article'); - if (articles.length === 1) { - contentEl = articles[0]; - } else if (articles.length > 1) { - let maxLen = 0; - articles.forEach(a => { - const len = textLen(a); - if (len > maxLen) { maxLen = len; contentEl = a; } - }); - } - if (!contentEl) contentEl = document.querySelector('[role="main"]'); - if (!contentEl) contentEl = document.querySelector('main'); - if (!contentEl) { - const candidates = document.querySelectorAll( - 'div[class*="content"], div[class*="article"], div[class*="post"], ' + - 'div[class*="entry"], div[class*="body"], div[id*="content"], ' + - 'div[id*="article"], div[id*="post"], section' - ); - let maxLen = 0; - candidates.forEach(c => { - const len = textLen(c); - if (len > maxLen) { maxLen = len; contentEl = c; } - }); - } - if (!contentEl || textLen(contentEl) < 200) contentEl = document.body; - - const clone = contentEl.cloneNode(true); - absolutizeTree(clone, window.location.href); - - const originalFrames = Array.from(contentEl.querySelectorAll('iframe')); - const clonedFrames = Array.from(clone.querySelectorAll('iframe')); - const clonedFrameByOriginal = new Map(); - originalFrames.forEach((frame, index) => { - const cloned = clonedFrames[index]; - if (cloned) clonedFrameByOriginal.set(frame, cloned); - }); - const allFrames = Array.from(document.querySelectorAll('iframe')); - const frameDescriptions = new Map(); - allFrames.forEach((frame, index) => frameDescriptions.set(frame, describeFrame(frame, index))); - const getFrameDescription = (frame, fallbackIndex) => frameDescriptions.get(frame) || describeFrame(frame, fallbackIndex); - result.diagnostics.frames = allFrames.map(frame => frameDescriptions.get(frame)); - - if (frameMode === 'same-origin' || frameMode === 'all-same-origin') { - allFrames.forEach((frame, index) => { - const insideContent = contentEl.contains(frame); - const cloned = insideContent ? clonedFrameByOriginal.get(frame) : null; - if (insideContent && !cloned) return; - const desc = getFrameDescription(frame, index); - if (!desc.sameOrigin || !desc.accessible) return; - try { - const doc = frame.contentDocument; - if (!doc?.body) return; - const frameBody = doc.body.cloneNode(true); - if (frameMode !== 'all-same-origin' && !insideContent && !shouldIncludeExternalFrame(frameBody)) return; - const section = buildFrameSection(frameBody, desc, frame.getAttribute('src') || ('#' + index)); - if (insideContent) cloned.replaceWith(section); - else clone.appendChild(section); - result.diagnostics.includedFrameCount += 1; - } catch {} - }); - } - - collectEmptyContainers(clone, 'main', window.location.href); - - const noise = 'nav, header, footer, aside, .sidebar, .nav, .menu, .footer, ' + - '.header, .comments, .comment, .ad, .ads, .advertisement, .social-share, ' + - '.related-posts, .newsletter, .cookie-banner, script, style, noscript, iframe'; - clone.querySelectorAll(noise).forEach(el => el.remove()); - - const stripWS = (s) => (s || '').replace(/\\s+/g, ''); - const dedup = (parent) => { - const children = Array.from(parent.children || []); - for (let i = children.length - 1; i >= 1; i--) { - const curRaw = children[i].textContent || ''; - const prevRaw = children[i - 1].textContent || ''; - const cur = stripWS(curRaw); - const prev = stripWS(prevRaw); - if (cur.length < 20 || prev.length < 20) continue; - if (cur === prev) { - const curSpaces = (curRaw.match(/ /g) || []).length; - const prevSpaces = (prevRaw.match(/ /g) || []).length; - if (curSpaces >= prevSpaces) children[i - 1].remove(); - else children[i].remove(); - } else if (prev.includes(cur) && cur.length / prev.length > 0.8) { - children[i].remove(); - } else if (cur.includes(prev) && prev.length / cur.length > 0.8) { - children[i - 1].remove(); - } - } - }; - dedup(clone); - clone.querySelectorAll('section, div').forEach(el => { - if (el.children && el.children.length > 2) dedup(el); - }); - - clone.querySelectorAll('img').forEach(img => { - const srcset = img.getAttribute('data-srcset') || ''; - const srcsetFirst = srcset.split(',')[0]?.trim().split(' ')[0] || ''; - const real = img.getAttribute('data-src') - || img.getAttribute('data-original') - || img.getAttribute('data-lazy-src') - || srcsetFirst; - if (real) img.setAttribute('src', absolutize(real, window.location.href)); - }); - - result.contentHtml = clone.innerHTML; - - const seen = new Set(); - clone.querySelectorAll('img').forEach(img => { - const src = img.getAttribute('src') || ''; - if (src && !src.startsWith('data:') && !seen.has(src)) { - seen.add(src); - result.imageUrls.push(src); - } - }); - - return result; - })() - `; -} - -function formatDiagnostics(data, networkEntries, captureSupported) { - const lines = []; - const diag = data?.diagnostics || {}; - lines.push('[web-fetch-browser diagnose]'); - lines.push(`url: ${diag.url || '-'}`); - lines.push(`frames: ${Array.isArray(diag.frames) ? diag.frames.length : 0}, included_same_origin: ${diag.includedFrameCount || 0}`); - for (const frame of (diag.frames || []).slice(0, 20)) { - lines.push(` [frame ${frame.index}] ${frame.sameOrigin ? 'same-origin' : 'cross-origin'} ${frame.accessible ? 'accessible' : 'blocked'} text=${frame.textLength || 0} ${frame.src || '-'}`); - } - if (Array.isArray(diag.emptyContainers) && diag.emptyContainers.length > 0) { - lines.push(`empty_containers: ${diag.emptyContainers.length}`); - for (const item of diag.emptyContainers.slice(0, 12)) { - const selector = `${item.tag}${item.id ? `#${item.id}` : ''}${item.className ? `.${String(item.className).trim().split(/\\s+/).filter(Boolean).join('.')}` : ''}`; - lines.push(` ${item.scope}: ${selector} (${item.url || '-'})`); - } - } - const interesting = networkEntries.filter(isInterestingNetworkEntry); - lines.push(`network_capture: ${captureSupported ? 'enabled' : 'unavailable'}, entries=${networkEntries.length}, api_like=${interesting.length}`); - for (const entry of interesting.slice(0, 20)) { - lines.push(` ${entry.method} ${entry.status || '-'} ${entry.contentType || '-'} ${entry.url}`); - } - return `${lines.join('\n')}\n`; -} - -const command = cli({ - site: 'web', - name: 'fetch-browser', - access: 'read', - description: 'Fetch any web page and export as Markdown', - strategy: Strategy.COOKIE, - navigateBefore: false, // we handle navigation ourselves - args: [ - { name: 'url', required: true, help: 'Any web page URL' }, - { name: 'output', default: './web-articles', help: 'Output directory' }, - { name: 'download-images', type: 'boolean', default: true, help: 'Download images locally' }, - { name: 'wait', type: 'int', default: 3, help: 'Seconds to wait after page load' }, - { name: 'wait-for', valueRequired: true, help: 'CSS selector to wait for in the main document or same-origin iframes' }, - { name: 'wait-until', default: 'domstable', choices: ['domstable', 'networkidle'], help: 'Readiness policy after navigation: domstable or networkidle' }, - { name: 'frames', default: 'same-origin', choices: ['same-origin', 'all-same-origin', 'none'], help: 'Iframe handling mode: relevant same-origin, all-same-origin, or none' }, - { name: 'diagnose', type: 'boolean', default: false, help: 'Print render diagnostics (frames, empty containers, XHR/API-like requests) to stderr' }, - { name: 'stdout', type: 'boolean', default: false, help: 'Print markdown to stdout instead of saving to a file' }, - ], - columns: ['title', 'author', 'publish_time', 'status', 'size', 'saved'], - func: async (page, kwargs, debug = false) => { - const url = kwargs.url; - const waitSeconds = kwargs.wait ?? 3; - const waitUntil = normalizeWaitUntil(kwargs['wait-until']); - const frameMode = normalizeFrameMode(kwargs.frames); - const shouldDiagnose = boolish(kwargs.diagnose) || debug || !!process.env.WEBCMD_VERBOSE; - const networkEntries = []; - const captureSupported = (waitUntil === 'networkidle' || shouldDiagnose) - ? await maybeStartNetworkCapture(page) - : false; - // Navigate to the target URL - await page.goto(url); - if (kwargs['wait-for']) { - const waitResult = await page.evaluate(buildWaitForSelectorAcrossFramesJs(String(kwargs['wait-for']), waitSeconds * 1000)); - if (waitResult?.invalidSelector) { - throw new Error(`Invalid --wait-for selector "${kwargs['wait-for']}": ${waitResult.error || 'querySelector failed'}`); - } - if (!waitResult?.ok) { - throw new Error(`Timed out waiting for selector "${kwargs['wait-for']}" in main document or same-origin iframes`); - } - } else if (waitUntil !== 'networkidle') { - await page.wait(waitSeconds); - } - if (waitUntil === 'networkidle') { - if (!captureSupported) { - throw new Error('Network capture is unavailable, so --wait-until networkidle cannot be satisfied'); - } - const idle = await waitForNetworkIdle(page, waitSeconds, networkEntries); - if (!idle?.ok) { - throw new Error(`Timed out waiting for network idle after ${waitSeconds}s`); - } - } - // Extract article content using browser-side heuristics - const data = await page.evaluate(buildRenderAwareExtractorJs({ frames: frameMode })); - if (captureSupported) await drainNetworkCapture(page, networkEntries); - if (shouldDiagnose) process.stderr.write(formatDiagnostics(data, networkEntries, captureSupported)); - // Determine Referer from URL for image downloads - let referer = ''; - try { - const parsed = new URL(url); - referer = parsed.origin + '/'; - } - catch { /* ignore */ } - const result = await downloadArticle({ - title: data?.title || 'untitled', - author: data?.author, - publishTime: data?.publishTime, - sourceUrl: url, - contentHtml: data?.contentHtml || '', - imageUrls: data?.imageUrls, - }, { - output: kwargs.output, - downloadImages: kwargs['download-images'], - imageHeaders: referer ? { Referer: referer } : undefined, - stdout: kwargs.stdout, - configureTurndown: (td) => { - td.addRule('preserveButtons', { - filter: (node) => node.nodeName === 'BUTTON', - replacement: (content) => content, - }); - }, - }); - // `--stdout` is a content-streaming mode. The markdown body already went - // to process.stdout inside downloadArticle(), so returning rows here - // would make Commander append table/JSON output to the same stdout - // stream and break piping. - return kwargs.stdout ? null : result; - }, -}); -export const __test__ = { - command, - buildRenderAwareExtractorJs, - buildWaitForSelectorAcrossFramesJs, - formatDiagnostics, - isInterestingNetworkEntry, - normalizeFrameMode, - normalizeWaitUntil, -}; diff --git a/clis/web/fetch.js b/clis/web/fetch.js deleted file mode 100644 index 113b6169..00000000 --- a/clis/web/fetch.js +++ /dev/null @@ -1 +0,0 @@ -import '@agentrhq/webcmd/fetch/command'; diff --git a/clis/web/test/fetch-browser.test.js b/clis/web/test/fetch-browser.test.js deleted file mode 100644 index 500a7ac3..00000000 --- a/clis/web/test/fetch-browser.test.js +++ /dev/null @@ -1,392 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { JSDOM } from 'jsdom'; - -const { mockDownloadArticle } = vi.hoisted(() => ({ - mockDownloadArticle: vi.fn(), -})); - -vi.mock('@agentrhq/webcmd/download/article-download', () => ({ - downloadArticle: mockDownloadArticle, -})); - -const { __test__ } = await import('../fetch-browser.js'); - -describe('web/fetch-browser stdout behavior', () => { - const read = __test__.command; - const extractedArticle = { - title: 'Example Article', - author: 'Author', - publishTime: '2026-04-22', - contentHtml: '

hello

', - imageUrls: ['https://example.com/a.jpg'], - diagnostics: { - url: 'https://example.com/article', - frames: [], - emptyContainers: [], - includedFrameCount: 0, - }, - }; - const page = { - goto: vi.fn().mockResolvedValue(undefined), - wait: vi.fn().mockResolvedValue(undefined), - evaluate: vi.fn().mockResolvedValue(extractedArticle), - startNetworkCapture: vi.fn().mockResolvedValue(true), - readNetworkCapture: vi.fn().mockResolvedValue([]), - }; - - beforeEach(() => { - vi.restoreAllMocks(); - vi.useRealTimers(); - mockDownloadArticle.mockReset(); - mockDownloadArticle.mockResolvedValue([{ - title: 'Example Article', - author: 'Author', - publish_time: '2026-04-22', - status: 'success', - size: '1 KB', - saved: '-', - }]); - page.goto.mockClear(); - page.wait.mockClear(); - page.evaluate.mockClear(); - page.evaluate.mockResolvedValue(extractedArticle); - page.startNetworkCapture.mockClear(); - page.startNetworkCapture.mockResolvedValue(true); - page.readNetworkCapture.mockClear(); - page.readNetworkCapture.mockResolvedValue([]); - }); - - it('returns null in --stdout mode so the CLI does not append result rows to stdout', async () => { - const result = await read.func(page, { - url: 'https://example.com/article', - output: '/tmp/out', - 'download-images': false, - stdout: true, - }); - - expect(result).toBeNull(); - expect(mockDownloadArticle).toHaveBeenCalledWith( - expect.objectContaining({ - title: 'Example Article', - sourceUrl: 'https://example.com/article', - }), - expect.objectContaining({ - output: '/tmp/out', - stdout: true, - }), - ); - expect(page.evaluate.mock.calls[0]?.[0]).toContain('const frameMode = "same-origin"'); - }); - - it('still returns the saved-row payload when writing to disk', async () => { - const rows = [{ title: 'Example Article', saved: '/tmp/out/Example Article/example.md' }]; - mockDownloadArticle.mockResolvedValue(rows); - - const result = await read.func(page, { - url: 'https://example.com/article', - output: '/tmp/out', - 'download-images': false, - stdout: false, - }); - - expect(result).toBe(rows); - }); - - it('waits for a selector in the main document or same-origin iframes before extracting', async () => { - page.evaluate - .mockResolvedValueOnce({ ok: true, scope: 'iframe', url: 'https://example.com/frame' }) - .mockResolvedValueOnce(extractedArticle); - - await read.func(page, { - url: 'https://example.com/article', - output: '/tmp/out', - 'download-images': false, - 'wait-for': '#gridDatas li', - wait: 7, - stdout: false, - }); - - expect(page.wait).not.toHaveBeenCalled(); - expect(page.evaluate).toHaveBeenCalledTimes(2); - expect(page.evaluate.mock.calls[0]?.[0]).toContain('"#gridDatas li"'); - expect(page.evaluate.mock.calls[0]?.[0]).toContain('sameOriginFrameDocs'); - expect(page.evaluate.mock.calls[1]?.[0]).toContain('const frameMode = "same-origin"'); - }); - - it('throws a clear error when --wait-for times out', async () => { - page.evaluate.mockResolvedValueOnce({ ok: false, timedOut: true, selector: '#missing' }); - - await expect(read.func(page, { - url: 'https://example.com/article', - output: '/tmp/out', - 'download-images': false, - 'wait-for': '#missing', - wait: 1, - stdout: false, - })).rejects.toThrow('Timed out waiting for selector "#missing"'); - - expect(mockDownloadArticle).not.toHaveBeenCalled(); - }); - - it('starts network capture and writes diagnostics in diagnose mode', async () => { - const stderr = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); - page.readNetworkCapture.mockResolvedValueOnce([{ - method: 'POST', - url: 'https://example.com/api/data', - responseStatus: 200, - responseContentType: 'application/json', - responsePreview: '{"ok":true}', - }]); - - await read.func(page, { - url: 'https://example.com/article', - output: '/tmp/out', - 'download-images': false, - diagnose: true, - wait: 0, - stdout: false, - }); - - expect(page.startNetworkCapture).toHaveBeenCalledWith(''); - expect(page.readNetworkCapture).toHaveBeenCalledTimes(1); - expect(stderr).toHaveBeenCalledWith(expect.stringContaining('[web-fetch-browser diagnose]')); - expect(stderr).toHaveBeenCalledWith(expect.stringContaining('POST 200 application/json https://example.com/api/data')); - }); - - it('passes --frames none into the extractor', async () => { - await read.func(page, { - url: 'https://example.com/article', - output: '/tmp/out', - 'download-images': false, - frames: 'none', - stdout: false, - }); - - expect(page.evaluate.mock.calls[0]?.[0]).toContain('const frameMode = "none"'); - }); - - it('passes --frames all-same-origin into the extractor', async () => { - await read.func(page, { - url: 'https://example.com/article', - output: '/tmp/out', - 'download-images': false, - frames: 'all-same-origin', - stdout: false, - }); - - expect(page.evaluate.mock.calls[0]?.[0]).toContain('const frameMode = "all-same-origin"'); - }); - - it('fails fast when --wait-until networkidle is requested but capture is unavailable', async () => { - page.startNetworkCapture.mockResolvedValue(false); - - await expect(read.func(page, { - url: 'https://example.com/article', - output: '/tmp/out', - 'download-images': false, - 'wait-until': 'networkidle', - wait: 2, - stdout: false, - })).rejects.toThrow('Network capture is unavailable'); - - expect(page.wait).not.toHaveBeenCalled(); - expect(mockDownloadArticle).not.toHaveBeenCalled(); - }); - - it('fails fast when network traffic never settles before the networkidle timeout', async () => { - vi.useFakeTimers(); - page.readNetworkCapture.mockResolvedValue([{ - method: 'POST', - url: 'https://example.com/api/data', - responseStatus: 200, - responseContentType: 'application/json', - responsePreview: '{"ok":true}', - }]); - - const pending = expect(read.func(page, { - url: 'https://example.com/article', - output: '/tmp/out', - 'download-images': false, - 'wait-until': 'networkidle', - wait: 1, - stdout: false, - })).rejects.toThrow('Timed out waiting for network idle after 1s'); - - await vi.advanceTimersByTimeAsync(2000); - - await pending; - expect(mockDownloadArticle).not.toHaveBeenCalled(); - }); -}); - -describe('web/fetch-browser render-aware helpers', () => { - it('merges accessible same-origin iframe bodies into the extracted HTML', () => { - const dom = new JSDOM(` -
-

Shell

- -
- `, { url: 'https://example.com/main.html', runScripts: 'outside-only' }); - const frame = dom.window.document.querySelector('iframe'); - frame.contentDocument.open(); - frame.contentDocument.write('
Name

Station A 42

'); - frame.contentDocument.close(); - - const result = dom.window.eval(__test__.buildRenderAwareExtractorJs({ frames: 'same-origin' })); - - expect(result.diagnostics.includedFrameCount).toBe(1); - expect(result.contentHtml).toContain('data-webcmd-iframe-source="https://example.com/frame.html"'); - expect(result.contentHtml).toContain('localized text iframe: https://example.com/frame.html'); - expect(result.contentHtml).toContain('Station A 42'); - expect(result.diagnostics.emptyContainers).toEqual(expect.arrayContaining([ - expect.objectContaining({ scope: 'iframe', id: 'gridDatas', url: 'https://example.com/frame.html' }), - ])); - expect(result.diagnostics.emptyContainers.every(item => item.scope === 'iframe')).toBe(true); - }); - - it('merges readable same-origin iframes outside the selected content element', () => { - const dom = new JSDOM(` -
-

Main Article

-

${'Main content '.repeat(30)}

-
- - `, { url: 'https://example.com/main.html', runScripts: 'outside-only' }); - const frame = dom.window.document.querySelector('iframe'); - frame.contentDocument.open(); - frame.contentDocument.write(`

Outside Frame

${'Frame data '.repeat(12)}

`); - frame.contentDocument.close(); - - const result = dom.window.eval(__test__.buildRenderAwareExtractorJs({ frames: 'same-origin' })); - - expect(result.diagnostics.includedFrameCount).toBe(1); - expect(result.contentHtml).toContain('data-webcmd-iframe-source="https://example.com/outside.html"'); - expect(result.contentHtml).toContain('Outside Frame'); - expect(result.contentHtml).toContain('Frame data'); - }); - - it('keeps short data-like iframes outside the selected content element', () => { - const dom = new JSDOM(` -
-

Main Article

-

${'Main content '.repeat(30)}

-
- - `, { url: 'https://example.com/main.html', runScripts: 'outside-only' }); - const frame = dom.window.document.querySelector('iframe'); - frame.contentDocument.open(); - frame.contentDocument.write('
water level
42
'); - frame.contentDocument.close(); - - const result = dom.window.eval(__test__.buildRenderAwareExtractorJs({ frames: 'same-origin' })); - - expect(result.diagnostics.includedFrameCount).toBe(1); - expect(result.contentHtml).toContain('42'); - expect(result.diagnostics.emptyContainers).toEqual(expect.arrayContaining([ - expect.objectContaining({ scope: 'iframe', id: 'gridDatas', url: 'https://example.com/data.html' }), - ])); - expect(result.diagnostics.emptyContainers.every(item => item.scope === 'iframe')).toBe(true); - }); - - it('skips short non-structural iframes outside the selected content element', () => { - const dom = new JSDOM(` -
-

Main Article

-

${'Main content '.repeat(30)}

-
- - `, { url: 'https://example.com/main.html', runScripts: 'outside-only' }); - const frame = dom.window.document.querySelector('iframe'); - frame.contentDocument.open(); - frame.contentDocument.write('

tiny note

'); - frame.contentDocument.close(); - - const result = dom.window.eval(__test__.buildRenderAwareExtractorJs({ frames: 'same-origin' })); - - expect(result.diagnostics.includedFrameCount).toBe(0); - expect(result.contentHtml).not.toContain('tiny note'); - }); - - it('includes short non-structural iframes in all-same-origin mode', () => { - const dom = new JSDOM(` -
-

Main Article

-

${'Main content '.repeat(30)}

-
- - `, { url: 'https://example.com/main.html', runScripts: 'outside-only' }); - const frame = dom.window.document.querySelector('iframe'); - frame.contentDocument.open(); - frame.contentDocument.write('
Online: 42°C
'); - frame.contentDocument.close(); - - const result = dom.window.eval(__test__.buildRenderAwareExtractorJs({ frames: 'all-same-origin' })); - - expect(result.diagnostics.includedFrameCount).toBe(1); - expect(result.contentHtml).toContain('Online: 42°C'); - }); - - it('marks API-like network entries as interesting and ignores static assets', () => { - expect(__test__.isInterestingNetworkEntry({ - method: 'POST', - url: 'https://example.com/GJZ/Ajax/Publish.ashx', - status: 200, - contentType: 'text/html', - size: 100, - bodyTruncated: false, - })).toBe(true); - expect(__test__.isInterestingNetworkEntry({ - method: 'POST', - url: 'https://example.com/GJZ/Ajax/Publish.ashx', - status: 200, - contentType: 'application/json', - size: 100, - bodyTruncated: false, - })).toBe(true); - expect(__test__.isInterestingNetworkEntry({ - method: 'GET', - url: 'https://example.com/app.js', - status: 200, - contentType: 'application/javascript', - size: 100, - bodyTruncated: false, - })).toBe(false); - }); - - it('formats frame and XHR diagnostics for shell pages', () => { - const output = __test__.formatDiagnostics({ - diagnostics: { - url: 'https://example.com/main.html', - includedFrameCount: 1, - frames: [{ - index: 0, - src: 'https://example.com/frame.html', - sameOrigin: true, - accessible: true, - textLength: 42, - }], - emptyContainers: [{ - scope: 'iframe', - url: 'https://example.com/frame.html', - tag: 'ul', - id: 'gridDatas', - className: '', - }], - }, - }, [{ - method: 'POST', - url: 'https://example.com/GJZ/Ajax/Publish.ashx', - status: 200, - contentType: 'application/json', - size: 64, - bodyTruncated: false, - }], true); - - expect(output).toContain('frames: 1, included_same_origin: 1'); - expect(output).toContain('[frame 0] same-origin accessible text=42 https://example.com/frame.html'); - expect(output).toContain('iframe: ul#gridDatas (https://example.com/frame.html)'); - expect(output).toContain('POST 200 application/json https://example.com/GJZ/Ajax/Publish.ashx'); - }); -}); diff --git a/docs/cli-reference.mdx b/docs/cli-reference.mdx index 3e5bee3d..1f653638 100644 --- a/docs/cli-reference.mdx +++ b/docs/cli-reference.mdx @@ -35,11 +35,11 @@ webcmd wikipedia summary "Command-line interface" webcmd pubmed search "agentic browser automation" --limit 5 -f json ``` -Agents should prefer existing adapters before raw browser exploration. For search and fetch tasks, use the bundled `smart-search` skill: it tries fetch-based search first, browser fetch second, and search adapters last. +Agents should prefer existing adapters before raw browser exploration. For search and fetch tasks, use the bundled `smart-search` skill: it tries fetch-based search first, explicit browser Sessions after eligible fetch failures, and search adapters last. ## Search and Fetch -Use `webcmd web fetch` for direct URL fetches and fetch-first web search. It runs locally and does not launch a browser, including when Webcmd is configured for hosted mode. +Use `webcmd web fetch` for direct URL fetches and fetch-first web search. It is built into the CLI, so it works on a fresh install with no plugins. ```bash webcmd web fetch --url https://example.com/article @@ -47,15 +47,33 @@ webcmd web fetch --url "https://duckduckgo.com/html/?q=agentic%20browser%20autom webcmd web fetch --url "https://www.bing.com/search?q=agentic%20browser%20automation" ``` -Use `webcmd web fetch-browser` only when `web fetch` reports that browser rendering is required or the page is blocked: +`web fetch` tries plain HTTP first, then browser-impersonating TLS clients. It remains local in both modes and never opens a browser. If a page requires browser rendering or blocks non-browser fetches, it returns `FETCH_REQUIRES_BROWSER` or `FETCH_BLOCKED`; only those codes permit explicit browser fallback. + +For either code, create one Session, navigate with `browser run`, inspect with a read snapshot, reuse the Session for allowed browser work, and close it: ```bash -webcmd web fetch-browser --url https://example.com/app-shell +webcmd --profile work session create +# Copy the returned full ID: +# session_7d8f2c10-4a11-4f3e-9c22-1b6de0a91f45 + +webcmd --profile work \ + --session session_7d8f2c10-4a11-4f3e-9c22-1b6de0a91f45 \ + browser run --stdin <<'JS' +await page.goto('https://example.com'); +return { url: page.url(), title: await page.title() }; +JS + +webcmd --profile work \ + --session session_7d8f2c10-4a11-4f3e-9c22-1b6de0a91f45 \ + browser snapshot --snapshot-mode read + +webcmd --profile work session close \ + session_7d8f2c10-4a11-4f3e-9c22-1b6de0a91f45 ``` -The old `web read` command has been renamed to `web fetch-browser`. +Local browser commands use Cloak. Hosted browser commands use Webcmd Cloud and Browser Use; `web fetch` still runs locally. -## Local Browser Programs +## Browser Programs Create an opaque session before raw browser work. Profiles hold cookie/auth state; sessions are browser workspaces within that profile. Adapter commands @@ -110,7 +128,7 @@ into adapter modules. | `doctor` | Diagnose browser bridge and daemon connectivity. | | `daemon` | Manage the local Webcmd daemon: status, stop, and restart. | | `browser` | Agent-facing browser runtime for exploration and verification. | -| `web` | Local URL fetch and browser-backed page fetch helpers. | +| `web` | Local URL fetch helpers. | | `profile` | List, rename, and select browser runtime profiles. | | `auth` | Inspect website login status, and refresh logged-in site sessions. | | `plugin` | Install, update, list, create, and uninstall plugins. | diff --git a/scripts/check-package-bin.mjs b/scripts/check-package-bin.mjs index 01c40594..5b583627 100644 --- a/scripts/check-package-bin.mjs +++ b/scripts/check-package-bin.mjs @@ -33,6 +33,22 @@ function run(command, args, opts = {}) { return result; } +function output(result) { + return `${result.stdout}${result.stderr}`; +} + +function requireOutput(result, text, label) { + if (!output(result).includes(text)) { + fail(`${label} did not include ${JSON.stringify(text)}:\n${output(result).trim()}`); + } +} + +function rejectOutput(result, text, label) { + if (output(result).includes(text)) { + fail(`${label} unexpectedly included ${JSON.stringify(text)}:\n${output(result).trim()}`); + } +} + function parseNpmJsonArray(stdout) { const text = stdout.trim(); const jsonStart = text.lastIndexOf('\n['); @@ -70,11 +86,14 @@ try { } const packedPaths = new Set(packData.files.map((file) => file.path)); - for (const prefix of ['clis/', 'plugins/']) { + for (const prefix of ['clis/', 'plugins/', 'clis/web/', 'src/fetch/browser', 'dist/src/fetch/browser']) { if ([...packedPaths].some((packedPath) => packedPath.startsWith(prefix))) { fail(`packed tarball contains adapter source: ${prefix}`); } } + if ([...packedPaths].some((packedPath) => packedPath.includes('fetch-browser'))) { + fail('packed tarball contains fetch-browser artifact'); + } if (packedPaths.has('scripts/fetch-adapters.js')) { fail('packed tarball contains the retired adapter fetch lifecycle'); } @@ -87,6 +106,16 @@ try { const tarball = path.join(tmp, packData.filename); const prefix = path.join(tmp, 'prefix'); run('npm', ['install', '-g', tarball, '--prefix', prefix, '--ignore-scripts']); + const isolatedHome = path.join(tmp, 'home'); + const isolatedConfig = path.join(tmp, 'config'); + fs.mkdirSync(isolatedHome); + fs.mkdirSync(isolatedConfig); + const installedEnv = { + ...process.env, + HOME: isolatedHome, + USERPROFILE: isolatedHome, + WEBCMD_CONFIG_DIR: isolatedConfig, + }; for (const [name] of binEntries) { const binPath = process.platform === 'win32' @@ -95,8 +124,39 @@ try { if (!fs.existsSync(binPath)) { fail(`global install did not create executable: ${binPath}`); } - run(binPath, ['--version'], { cwd: tmp }); + run(binPath, ['--version'], { cwd: tmp, env: installedEnv }); + const fetchHelp = run(binPath, ['web', 'fetch', '--help'], { cwd: tmp, env: installedEnv }); + for (const option of ['--url', '--timeout', '--max-chars', '--allow-private']) { + requireOutput(fetchHelp, option, 'web fetch --help'); + } + const webHelp = run(binPath, ['web', '--help'], { cwd: tmp, env: installedEnv }); + requireOutput(webHelp, 'fetch', 'web --help'); + rejectOutput(webHelp, 'fetch-browser', 'web --help'); + const list = run(binPath, ['list', '-f', 'json'], { cwd: tmp, env: installedEnv }); + const listedCommands = JSON.parse(list.stdout); + if (listedCommands.filter((command) => command.command === 'web/fetch').length !== 1) { + fail(`list -f json did not contain exactly one web/fetch:\n${list.stdout.trim()}`); + } + if (listedCommands.some((command) => command.command === 'web/fetch-browser')) { + fail(`list -f json unexpectedly contained web/fetch-browser:\n${list.stdout.trim()}`); + } + const completions = run(binPath, ['--get-completions', '--cursor', '2', 'web'], { cwd: tmp, env: installedEnv }); + requireOutput(completions, 'fetch', '--get-completions --cursor 2 web'); + rejectOutput(completions, 'fetch-browser', '--get-completions --cursor 2 web'); } + + const packageRoot = run('npm', ['root', '-g', '--prefix', prefix]).stdout.trim(); + const installedPackagePath = path.join(packageRoot, pkg.name); + const fetchExport = pkg.exports?.['./fetch/command']; + if (typeof fetchExport !== 'string') fail('package.json has no ./fetch/command export'); + const installedFetchCommand = path.join(installedPackagePath, fetchExport); + if (!fs.existsSync(installedFetchCommand)) { + fail(`installed package is missing fetch command export: ${installedFetchCommand}`); + } + run(process.execPath, ['--input-type=module', '--eval', `import(${JSON.stringify(new URL(`file://${installedFetchCommand}`).href)})`], { + cwd: tmp, + env: installedEnv, + }); } finally { fs.rmSync(tmp, { recursive: true, force: true }); } diff --git a/scripts/check-plugin-command-parity.mjs b/scripts/check-plugin-command-parity.mjs index cefb15c1..bf0b6540 100644 --- a/scripts/check-plugin-command-parity.mjs +++ b/scripts/check-plugin-command-parity.mjs @@ -10,6 +10,8 @@ const core = read('cli-manifest.json'); const frozen = read('test/fixtures/core-cli-manifest-v0.5.3.json'); const coreKeys = new Set(core.map(key)); const pluginByKey = new Map(plugins.map(entry => [key(entry), entry])); +// This command was intentionally removed rather than migrated to a plugin. +const intentionallyRemoved = new Set(['web/fetch-browser']); // `args` is checked separately: a migrated command may still gain new optional // flags, so strict deep-equality would forbid ordinary feature work rather than // the regression this guards against. @@ -24,6 +26,7 @@ const issues = []; for (const expected of frozen) { const command = key(expected); if (coreKeys.has(command)) continue; + if (intentionallyRemoved.has(command)) continue; const actual = pluginByKey.get(command); if (!actual) { issues.push(`${command} is missing from plugin-command-manifest.json`); diff --git a/scripts/silent-column-drop-baseline.json b/scripts/silent-column-drop-baseline.json index d9ef6cf4..aaf3b402 100644 --- a/scripts/silent-column-drop-baseline.json +++ b/scripts/silent-column-drop-baseline.json @@ -394,27 +394,6 @@ "y" ] }, - { - "command": "web/fetch-browser", - "file": "clis/web/fetch-browser.js", - "missing": [ - "accessible", - "index", - "sameOrigin", - "src", - "textLength" - ] - }, - { - "command": "web/fetch-browser", - "file": "clis/web/fetch-browser.js", - "missing": [ - "bodyTruncated", - "contentType", - "method", - "url" - ] - }, { "command": "yahoo-finance/quote", "file": "plugins/yahoo-finance/quote.js", diff --git a/scripts/typed-error-lint-baseline.json b/scripts/typed-error-lint-baseline.json index df711138..facb7a41 100644 --- a/scripts/typed-error-lint-baseline.json +++ b/scripts/typed-error-lint-baseline.json @@ -527,38 +527,6 @@ "text": "const screenName = user?.legacy?.screen_name || user?.core?.screen_name || 'unknown';", "occurrence": 0 }, - { - "rule": "silent-sentinel", - "command": "web/fetch-browser", - "file": "clis/web/fetch-browser.js", - "line": 378, - "text": "lines.push(` [frame ${frame.index}] ${frame.sameOrigin ? 'same-origin' : 'cross-origin'} ${frame.accessible ? 'accessible' : 'blocked'} text=${frame.textLength || 0} ${frame.src || '-'}`);", - "occurrence": 0 - }, - { - "rule": "silent-sentinel", - "command": "web/fetch-browser", - "file": "clis/web/fetch-browser.js", - "line": 390, - "text": "lines.push(` ${entry.method} ${entry.status || '-'} ${entry.contentType || '-'} ${entry.url}`);", - "occurrence": 0 - }, - { - "rule": "silent-sentinel", - "command": "web/fetch-browser", - "file": "clis/web/fetch-browser.js", - "line": 384, - "text": "lines.push(` ${item.scope}: ${selector} (${item.url || '-'})`);", - "occurrence": 0 - }, - { - "rule": "silent-sentinel", - "command": "web/fetch-browser", - "file": "clis/web/fetch-browser.js", - "line": 375, - "text": "lines.push(`url: ${diag.url || '-'}`);", - "occurrence": 0 - }, { "rule": "silent-sentinel", "command": "yollomi/edit", diff --git a/skills/smart-search/SKILL.md b/skills/smart-search/SKILL.md index 2955c175..412b3712 100644 --- a/skills/smart-search/SKILL.md +++ b/skills/smart-search/SKILL.md @@ -11,7 +11,7 @@ Use live fetch results, command metadata, and command help. Do not infer command Do not use this skill for plugin inventory, plugin management, or listing available extensions. Marketplace commands appear here only to find and install search-capable adapters needed for the current search/fetch task. -Cost order is mandatory when the request does not name a site: `webcmd web fetch` first, `webcmd web fetch-browser` second, search adapters last. Do not call search adapters until plain HTTP/TLS fetch and browser fetch cannot satisfy the task. +Cost order is mandatory when the request does not name a site: `webcmd web fetch` first, search adapters last. `web fetch` runs locally in both modes and never opens a browser. Do not call search adapters until it has failed. When the request does name a site or community, take the site-native fast path below instead. @@ -35,21 +35,37 @@ Prefer primary sources, official docs, and direct content over search snippets. ## Direct URL -For a supplied HTTP(S) URL, fetch it first: +For a supplied HTTP(S) URL, fetch it: ```bash webcmd web fetch --url ``` -Only when the structured error code is `FETCH_BLOCKED` or `FETCH_REQUIRES_BROWSER`, use: +Try fetch once. Only `FETCH_BLOCKED` or `FETCH_REQUIRES_BROWSER` permits browser fallback; otherwise report the returned failure rather than retrying the URL. + +For browser fallback, create one Session, navigate the failed URL, inspect it, reuse that Session for allowed fallbacks, then close it. Local browser commands use Cloak; hosted browser commands use Webcmd Cloud and Browser Use. `web fetch` remains local in both modes. ```bash -webcmd web fetch-browser --url +webcmd --profile work session create +# Copy the returned full ID: +# session_7d8f2c10-4a11-4f3e-9c22-1b6de0a91f45 + +webcmd --profile work \ + --session session_7d8f2c10-4a11-4f3e-9c22-1b6de0a91f45 \ + browser run --stdin <<'JS' +await page.goto('https://example.com'); +return { url: page.url(), title: await page.title() }; +JS + +webcmd --profile work \ + --session session_7d8f2c10-4a11-4f3e-9c22-1b6de0a91f45 \ + browser snapshot --snapshot-mode read + +webcmd --profile work session close \ + session_7d8f2c10-4a11-4f3e-9c22-1b6de0a91f45 ``` -Do not escalate on message prose and do not make `web fetch` launch a browser. - -If direct fetch is rate-limited, blocked, CAPTCHA-gated, login-gated, geo-gated, or returns unusable extracted text, report that state. Only browser-escalate for `FETCH_BLOCKED` or `FETCH_REQUIRES_BROWSER`. +If the fetch is rate-limited, login-gated, geo-gated, or returns unusable extracted text, report that state rather than retrying the same URL. ## Fetch-first web search @@ -67,7 +83,7 @@ Query terms that collide with everyday English (`puppeteer`, `playwright`, `rust Extract useful result URLs from the fetched page and then fetch the target pages with `webcmd web fetch`. Search snippets and result titles are discovery only, not evidence. A page that yields zero usable result URLs is a failed search, not a search with no results: move to the next engine. -If the search-engine result page itself needs browser rendering, use at most one browser fetch for a search results page before trying another search engine. A recognised block, CAPTCHA, or challenge page retires that engine for this request: do not re-fetch variants of the same engine. Do not jump to adapters because one engine blocked, unless the request names a site. +If the search-engine result page returns `FETCH_BLOCKED` or `FETCH_REQUIRES_BROWSER`, use the Session workflow once within the browser Session budget. A recognised block, CAPTCHA, or challenge page retires that engine for this request: do not re-fetch variants of the same engine. Do not jump to adapters because one engine blocked, unless the request names a site. ## Fetch evidence @@ -77,13 +93,13 @@ Fetch up to three result URLs by default (five for a broad comparison): webcmd web fetch --url ``` -Use up to two browser fetches by default, only when target-page `web fetch` returns `FETCH_BLOCKED` or `FETCH_REQUIRES_BROWSER`. Cite or link the source URL with substantive claims. +For `FETCH_BLOCKED` or `FETCH_REQUIRES_BROWSER`, use the Session workflow above if the browser Session budget permits. Cite or link the source URL with substantive claims. If fetch is rate-limited, auth-gated, CAPTCHA-gated, bot-detected, quota-limited, or geo-blocked, do not loop. Try another relevant URL/source when available; otherwise report the blocker. ## Adapter fallback -On the site-named fast path, discover adapters first. Otherwise, only after fetch-first search, target-page fetch, and allowed browser fetches fail or are insufficient, discover search adapters: +On the site-named fast path, discover adapters first. Otherwise, only after fetch-first search, target-page fetch, and allowed browser Session fallbacks fail or are insufficient, discover search adapters: ```bash webcmd list --tag search -f json @@ -120,7 +136,7 @@ Do not add custom marketplaces in this workflow. In hosted mode, only verified h - One fetched search-engine page by default; second if weak/blocked; third only if the first two fail. - Up to five candidate commands before choosing. - Three URLs by default; five only for broad comparison. -- Two browser fetches by default. +- Two browser Sessions/URLs by default; reuse one Session for allowed browser fallbacks. - One adapter search by default; second only for weakness or corroboration. - Do not retry the same blocked command more than once. diff --git a/skills/webcmd-browser/SKILL.md b/skills/webcmd-browser/SKILL.md index 3bc5a7c0..1049dd24 100644 --- a/skills/webcmd-browser/SKILL.md +++ b/skills/webcmd-browser/SKILL.md @@ -30,7 +30,7 @@ Until `doctor` is green, browser commands may fail. Registry and plugin discover ## Session lifecycle -- Create an opaque browser session before raw browser work: `webcmd session create -f json`. +- Create an opaque browser session before raw browser work: `webcmd --profile session create`. - Raw browser commands require that ID at the root: `webcmd --session browser ...`; the old positional session form is retired. - Profiles are cookie jars and auth scope; sessions are browser workspaces/windows within a profile. Parallel agents use separate sessions. - `webcmd session list` shows sessions and their handoff/runtime state; close finished work with `webcmd session close `. Close is blocked while that Session has a live handoff. @@ -39,6 +39,28 @@ Until `doctor` is green, browser commands may fail. Registry and plugin discover - `webcmd --session browser bind --page ` explicitly attaches the session to an existing page. - If the user manually signs in or changes the visible tab, re-bind or inspect with a fresh snapshot before continuing. +For a `FETCH_BLOCKED` or `FETCH_REQUIRES_BROWSER` fallback, use one Session for the browser portion, preserve its complete opaque ID, and close it in cleanup. Local browser commands use Cloak; hosted browser commands use Webcmd Cloud and Browser Use. `web fetch` remains local in both modes and never opens a browser. + +```bash +webcmd --profile work session create +# Copy the returned full ID: +# session_7d8f2c10-4a11-4f3e-9c22-1b6de0a91f45 + +webcmd --profile work \ + --session session_7d8f2c10-4a11-4f3e-9c22-1b6de0a91f45 \ + browser run --stdin <<'JS' +await page.goto('https://example.com'); +return { url: page.url(), title: await page.title() }; +JS + +webcmd --profile work \ + --session session_7d8f2c10-4a11-4f3e-9c22-1b6de0a91f45 \ + browser snapshot --snapshot-mode read + +webcmd --profile work session close \ + session_7d8f2c10-4a11-4f3e-9c22-1b6de0a91f45 +``` + --- ## Command surface diff --git a/src/browser/run/runner.test.ts b/src/browser/run/runner.test.ts index da3e585e..216227ae 100644 --- a/src/browser/run/runner.test.ts +++ b/src/browser/run/runner.test.ts @@ -26,6 +26,7 @@ const playwrightServer = createRequire(import.meta.url)( let browser: Browser; let context: BrowserContext; let page: Page; +const describeWithChromium = fs.existsSync(chromium.executablePath()) ? describe : describe.skip; function sessionScope(pages: () => readonly Page[] = () => context.pages()) { return { @@ -48,6 +49,7 @@ function run(source: string, options = {}) { }, source, options); } +describeWithChromium('runBrowserProgram', () => { beforeAll(async () => { browser = await chromium.launch({ headless: true }); }); @@ -78,7 +80,6 @@ afterAll(async () => { await browser.close(); }); -describe('runBrowserProgram', () => { it('returns snapshotDiff by default after successful runs', async () => { const output = await run(` await page.setContent('
'); diff --git a/src/browser/sessions.test.ts b/src/browser/sessions.test.ts index bd8616bc..676a677d 100644 --- a/src/browser/sessions.test.ts +++ b/src/browser/sessions.test.ts @@ -69,7 +69,7 @@ describe('LocalBrowserSessionStore', () => { const statePath = path.join(baseDir, 'browser-sessions.json'); expect(fs.existsSync(statePath)).toBe(true); - expect(fs.statSync(statePath).mode & 0o777).toBe(0o600); + if (process.platform !== 'win32') expect(fs.statSync(statePath).mode & 0o777).toBe(0o600); expect(fs.readdirSync(baseDir).filter((name) => name.includes('.tmp'))).toEqual([]); }); diff --git a/src/build-manifest.test.ts b/src/build-manifest.test.ts index 403cae84..3887c26c 100644 --- a/src/build-manifest.test.ts +++ b/src/build-manifest.test.ts @@ -6,6 +6,7 @@ import { cli, getRegistry, Strategy } from './registry.js'; import { ManifestImportError, buildManifestArtifacts, + coreCommandEntries, diffRemovedEntries, findManifestMetadataIssues, loadManifestEntries, @@ -36,6 +37,22 @@ describe('manifest helper rules', () => { expect(serializeManifest([])).toBe('[]\n'); }); + it('serializes web fetch as the single client-owned core command', async () => { + const entries = await coreCommandEntries(); + + expect(entries).toHaveLength(1); + expect(entries).toEqual([ + expect.objectContaining({ + site: 'web', + name: 'fetch', + clientOwned: true, + packageExport: './fetch/command', + }), + ]); + expect(entries[0]).not.toHaveProperty('modulePath'); + expect(entries[0]).not.toHaveProperty('sourceFile'); + }); + it('skips TS files that do not register a cli', () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'webcmd-manifest-')); tempDirs.push(dir); diff --git a/src/build-manifest.ts b/src/build-manifest.ts index 5cb93350..8a20d7b3 100644 --- a/src/build-manifest.ts +++ b/src/build-manifest.ts @@ -111,7 +111,10 @@ function isCliCommandValue(value: unknown, site: string): value is CliCommand { && Array.isArray(value.args); } -function toManifestEntry(cmd: CliCommand, modulePath: string, sourceFile?: string): ManifestEntry { +function toManifestEntry( + cmd: CliCommand, + adapterPath?: Pick, +): ManifestEntry { return { site: cmd.site, name: cmd.name, @@ -128,8 +131,8 @@ function toManifestEntry(cmd: CliCommand, modulePath: string, sourceFile?: strin ...(cmd.keywords?.length ? { keywords: [...cmd.keywords] } : {}), defaultFormat: cmd.defaultFormat, type: 'js', - modulePath, - sourceFile, + ...adapterPath, + ...(cmd.clientOwned ? { clientOwned: true } : {}), navigateBefore: cmd.navigateBefore, siteSession: cmd.siteSession, freshPage: cmd.freshPage, @@ -195,7 +198,7 @@ export async function loadManifestEntries( return true; }) .sort((a, b) => a.name.localeCompare(b.name)) - .map(cmd => toManifestEntry(cmd, modulePath, sourceRelative)); + .map(cmd => toManifestEntry(cmd, { modulePath, sourceFile: sourceRelative })); } catch (err) { throw new ManifestImportError(filePath, err); } @@ -247,8 +250,45 @@ export async function scanClisDir( return { entries, failures }; } +/** + * Sites the core package owns and ships in `dist/`, rather than as adapter + * files under `clis/`. `web` moved here so the whole fetch ladder installs by + * default instead of living in a plugin nobody had (#247). + * + * The value is the package subpath export whose import registers the site's + * commands. It must stay listed in package.json `exports`, which + * `package-exports.test.ts` verifies resolves to a real source file. + */ +const CORE_SITE_EXPORTS = new Map([['web', './fetch/command']]); + +/** + * Manifest entries for core-registered commands. + * + * These deliberately carry no `modulePath`/`sourceFile`: there is no adapter + * file under `clis/` to resolve, and claiming one would point every consumer at + * a path that does not exist in the published tarball. They carry + * `packageExport` instead, so a consumer that loads adapters by path can import + * the command from the package itself rather than guessing its layout. + */ +export async function coreCommandEntries( + importer: (moduleHref: string) => Promise = moduleHref => import(moduleHref), +): Promise { + await importer(pathToFileURL(path.join(PACKAGE_ROOT, 'src/fetch/command.ts')).href); + return [...getRegistry().values()] + .filter(cmd => cmd.clientOwned && CORE_SITE_EXPORTS.has(cmd.site)) + .sort((a, b) => a.site.localeCompare(b.site) || a.name.localeCompare(b.name)) + .map(cmd => ({ ...toManifestEntry(cmd), packageExport: CORE_SITE_EXPORTS.get(cmd.site)! })); +} + export async function buildManifest(): Promise { - return scanClisDir(LEGACY_CLIS_DIR); + const scanned = await scanClisDir(LEGACY_CLIS_DIR); + const core = await coreCommandEntries(); + const entriesByCommand = new Map(scanned.entries.map(entry => [`${entry.site}/${entry.name}`, entry])); + for (const entry of core) entriesByCommand.set(`${entry.site}/${entry.name}`, entry); + const entries = [...entriesByCommand.values()].sort( + (a, b) => a.site.localeCompare(b.site) || a.name.localeCompare(b.name), + ); + return { entries, failures: scanned.failures }; } export function serializeManifest(manifest: ManifestEntry[]): string { diff --git a/src/cli.test.ts b/src/cli.test.ts index 7cb27635..374a21b5 100644 --- a/src/cli.test.ts +++ b/src/cli.test.ts @@ -70,16 +70,20 @@ vi.mock('node:child_process', async () => { import { createProgram, findPackageRoot, loadAntigravityServe, normalizeVerifyRows, renderVerifyPreview, resolveBrowserVerifyInvocation, resolveSitemapAvailabilityForUrl, selectFreshByTimestamp } from './cli.js'; const realHome = process.env.HOME; +const realConfigDir = process.env.WEBCMD_CONFIG_DIR; let isolatedCliTestHome: string; beforeEach(() => { isolatedCliTestHome = fs.mkdtempSync(path.join(os.tmpdir(), 'webcmd-cli-home-')); process.env.HOME = isolatedCliTestHome; + process.env.WEBCMD_CONFIG_DIR = path.join(isolatedCliTestHome, '.webcmd'); }); afterEach(() => { if (realHome === undefined) delete process.env.HOME; else process.env.HOME = realHome; + if (realConfigDir === undefined) delete process.env.WEBCMD_CONFIG_DIR; + else process.env.WEBCMD_CONFIG_DIR = realConfigDir; fs.rmSync(isolatedCliTestHome, { recursive: true, force: true }); }); diff --git a/src/cli.ts b/src/cli.ts index 78cfc87f..c7cfab03 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -13,6 +13,9 @@ import { fileURLToPath, pathToFileURL } from 'node:url'; import { Command, Option } from 'commander'; import { findPackageRoot, getBuiltEntryCandidates } from './package-paths.js'; import { type CliCommand, getRegistry } from './registry.js'; +// Side-effect import: registers client-owned `web fetch` in the core registry +// so it reaches help, `list`, completions and manifests without a plugin. +import './fetch/command.js'; import { commandListPresentation, filterCommandsByTag, toPresentableCommand } from './command-presentation.js'; import { configureCompletionCommandSurface, configureListCommandSurface, configurePluginInstallSurface, configurePluginSearchSurface } from './builtin-command-surface.js'; import { render as renderOutput } from './output.js'; diff --git a/src/command-surface.ts b/src/command-surface.ts index e3456fe3..7bccf326 100644 --- a/src/command-surface.ts +++ b/src/command-surface.ts @@ -1,6 +1,6 @@ import { Command } from 'commander'; import { ArgumentError } from './errors.js'; -import type { Arg } from './registry.js'; +import type { Arg, CliCommand, CommandArgs } from './registry.js'; export const OUTPUT_FORMATS = ['table', 'plain', 'json', 'yaml', 'yml', 'md', 'markdown', 'csv'] as const; export const TRACE_MODES = ['off', 'on', 'retain-on-failure'] as const; @@ -246,6 +246,16 @@ export function coerceCommandArguments( return result; } +/** Apply the adapter's coercion and command-specific validation. */ +export function prepareCommandArgs( + cmd: CliCommand, + rawKwargs: CommandArgs, +): CommandArgs { + const kwargs = coerceCommandArguments(cmd.args, rawKwargs); + cmd.validateArgs?.(kwargs); + return kwargs; +} + export function parseOutputFormat(value: unknown): OutputFormat { // Preserve the long-standing local behavior: unknown format names flow to // output.ts, whose default switch branch renders a table. diff --git a/src/commanderAdapter.ts b/src/commanderAdapter.ts index 6ae9a433..bed9899c 100644 --- a/src/commanderAdapter.ts +++ b/src/commanderAdapter.ts @@ -14,8 +14,7 @@ import { Command } from 'commander'; import { log } from './logger.js'; import { type CliCommand, fullName, getRegistry } from './registry.js'; import { formatErrorEnvelope, render as renderOutput } from './output.js'; -import { executeCommand, prepareCommandArgs } from './execution.js'; -import { configureCommandSurface, parseOutputFormat } from './command-surface.js'; +import { configureCommandSurface, parseOutputFormat, prepareCommandArgs } from './command-surface.js'; import { commandHelpData, formatCommandHelpText, @@ -101,15 +100,17 @@ export function registerCommandToProgram( const formatExplicit = subCmd.getOptionValueSource('format') === 'cli'; if (verbose) process.env.WEBCMD_VERBOSE = '1'; const globals = typeof subCmd.optsWithGlobals === 'function' ? subCmd.optsWithGlobals() as Record : {}; - const result = await executeCommand(cmd, kwargs, verbose, { - prepared: true, - ...(typeof globals.profile === 'string' && globals.profile.trim() ? { profile: globals.profile.trim() } : {}), - ...(typeof globals.session === 'string' && globals.session.trim() ? { session: globals.session.trim() } : {}), - ...(typeof optionsRecord.trace === 'string' && optionsRecord.trace !== 'off' ? { trace: optionsRecord.trace } : {}), - ...(cmd.browser && typeof optionsRecord.window === 'string' ? { windowMode: optionsRecord.window } : {}), - ...(cmd.browser && typeof optionsRecord.siteSession === 'string' ? { siteSession: optionsRecord.siteSession } : {}), - ...(cmd.browser && typeof optionsRecord.keepTab === 'string' ? { keepTab: optionsRecord.keepTab } : {}), - }); + const result = cmd.clientOwned && cmd.browser === false + ? await cmd.func?.(kwargs, verbose) + : await (await import('./execution.js')).executeCommand(cmd, kwargs, verbose, { + prepared: true, + ...(typeof globals.profile === 'string' && globals.profile.trim() ? { profile: globals.profile.trim() } : {}), + ...(typeof globals.session === 'string' && globals.session.trim() ? { session: globals.session.trim() } : {}), + ...(typeof optionsRecord.trace === 'string' && optionsRecord.trace !== 'off' ? { trace: optionsRecord.trace } : {}), + ...(cmd.browser && typeof optionsRecord.window === 'string' ? { windowMode: optionsRecord.window } : {}), + ...(cmd.browser && typeof optionsRecord.siteSession === 'string' ? { siteSession: optionsRecord.siteSession } : {}), + ...(cmd.browser && typeof optionsRecord.keepTab === 'string' ? { keepTab: optionsRecord.keepTab } : {}), + }); if (result === null || result === undefined) { return; } @@ -130,6 +131,7 @@ export function registerCommandToProgram( elapsed: (now() - startTime) / 1000, source: fullName(resolved), footerExtra: resolved.footerExtra?.(kwargs), + ...(resolved.renderMarkdown ? { markdown: resolved.renderMarkdown } : {}), ...(runtime.stdout ? { stdout: runtime.stdout } : {}), }); } catch (err) { diff --git a/src/completion-shared.ts b/src/completion-shared.ts index f4a41caa..9d51ccdc 100644 --- a/src/completion-shared.ts +++ b/src/completion-shared.ts @@ -45,6 +45,7 @@ export const HOSTED_ROOT_HELP: RootHelpPresentation = { { name: 'list', description: 'List all available hosted CLI commands' }, { name: 'profile', description: 'Manage hosted browser profiles' }, { name: 'setup', description: 'Configure local or hosted mode' }, + { name: 'web', description: 'Fetch URLs locally without launching a browser' }, ], localOnlyCommands: [ { name: 'adapter', description: 'Manage adapters installed on this computer' }, diff --git a/src/execution.ts b/src/execution.ts index e03e6607..4443af72 100644 --- a/src/execution.ts +++ b/src/execution.ts @@ -40,7 +40,8 @@ import { isElectronApp } from './electron-apps.js'; import { probeCDP, resolveElectronEndpoint } from './launcher.js'; import { ObservationSession, exportObservationSession, type ObservationExportResult, type ObservationExportStatus } from './observation/index.js'; import { resolveAdapterSourcePath } from './adapter-source.js'; -import { coerceCommandArguments, TRACE_MODES, type TraceMode } from './command-surface.js'; +import { prepareCommandArgs, TRACE_MODES, type TraceMode } from './command-surface.js'; +export { prepareCommandArgs } from './command-surface.js'; import { clearDaemonRunContext, generateRunId, isUnknownOutcomeError, runWithDaemonRunContext } from './session-lease.js'; const _loadedModules = new Map>(); @@ -513,15 +514,6 @@ function exportTraceArtifact( } } -export function prepareCommandArgs( - cmd: CliCommand, - rawKwargs: CommandArgs, -): CommandArgs { - const kwargs = coerceCommandArguments(cmd.args, rawKwargs); - cmd.validateArgs?.(kwargs); - return kwargs; -} - /** * Runtime ceiling padding (seconds) added on top of the user's `--timeout`. * The adapter's polling loop typically uses the full user value; the padding diff --git a/src/fetch/classify.test.ts b/src/fetch/classify.test.ts index 7d4f42f7..1f44b9fb 100644 --- a/src/fetch/classify.test.ts +++ b/src/fetch/classify.test.ts @@ -6,5 +6,65 @@ describe('fetch classification', () => { expect(isChallengeResponse(403, { server: 'cloudflare' }, 'Just a moment...')).toBe(true); expect(isChallengeResponse(403, {}, 'forbidden')).toBe(false); }); + + it('does not treat healthy provider evidence as a challenge', () => { + expect(isChallengeResponse(200, { 'x-datadome': 'protected' }, '
real article
')).toBe(false); + expect(isChallengeResponse(200, { 'set-cookie': '__cf_bm=abc' }, '
real article
')).toBe(false); + expect(isChallengeResponse(200, { 'content-security-policy': 'script-src https://cdnjs.cloudflare.com' }, '
ok
')).toBe(false); + }); + it('recognizes script-heavy app shells', () => expect(isJavaScriptShell('
')).toBe(true)); + + // A CSP allow-list names third parties this page may load; it says nothing + // about who served the response. news.ycombinator.com returns a healthy 200 + // whose CSP mentions cdnjs.cloudflare.com and google.com/recaptcha (#264). + it('ignores content-security-policy allow-lists naming a CDN or captcha vendor', () => { + const csp = "default-src 'self'; script-src 'self' https://www.google.com/recaptcha/ https://cdnjs.cloudflare.com/"; + expect(isChallengeResponse(200, { 'content-security-policy': csp }, 'Hacker News')).toBe(false); + }); + + it('ignores report-to and link headers naming a challenge vendor', () => { + const headers = { 'report-to': '{"endpoints":[{"url":"https://report.cloudflare.com/"}]}', link: '; rel=preconnect' }; + expect(isChallengeResponse(200, headers, 'Real content')).toBe(false); + }); + + // example.com is a plain static page fronted by Cloudflare. `server: cloudflare` + // on a 200 that was actually served is not a challenge (#283). + it('does not flag a Cloudflare-fronted 200 whose body is a real page', () => { + const headers = { server: 'cloudflare', 'cf-cache-status': 'HIT', 'content-type': 'text/html' }; + expect(isChallengeResponse(200, headers, '

Example Domain

')).toBe(false); + }); + + it('still flags a managed-challenge interstitial served with a 200', () => { + const headers = { server: 'cloudflare', 'cf-mitigated': 'challenge' }; + expect(isChallengeResponse(200, headers, 'Just a moment...')).toBe(true); + }); + + // cf-mitigated only ever appears on an actual mitigation, so it stands alone + // even when the body has been withheld. + it('treats challenge-specific headers as evidence without a body marker', () => { + expect(isChallengeResponse(403, { 'cf-mitigated': 'challenge' }, '')).toBe(true); + expect(isChallengeResponse(429, { 'x-datadome': 'protected' }, '')).toBe(true); + expect(isChallengeResponse(503, { 'cf-chl-bypass': '1' }, '')).toBe(true); + }); + + it('treats a challenge cookie as evidence', () => { + expect(isChallengeResponse(403, { 'set-cookie': '__cf_bm=abc; Path=/' }, 'blocked')).toBe(true); + expect(isChallengeResponse(200, { 'set-cookie': 'session=abc; Path=/' }, 'real page')).toBe(false); + }); + + // A bare CDN name corroborates an already-suspicious status, but never decides + // on its own — that is the difference between #283 and a genuine block. + it('lets a bare CDN name corroborate a non-200 block', () => { + expect(isChallengeResponse(403, { server: 'cloudflare' }, 'Access denied')).toBe(true); + expect(isChallengeResponse(200, { server: 'akamai' }, 'Real page content')).toBe(false); + }); + + it('does not flag a healthy 200 with no challenge evidence anywhere', () => { + expect(isChallengeResponse(200, { 'content-type': 'text/html' }, 'Hello')).toBe(false); + }); + + it('still catches a body-level captcha wall on a 403', () => { + expect(isChallengeResponse(403, {}, '
verify you are human
')).toBe(true); + }); }); diff --git a/src/fetch/classify.ts b/src/fetch/classify.ts index 4051c13c..fa336a6f 100644 --- a/src/fetch/classify.ts +++ b/src/fetch/classify.ts @@ -1,8 +1,41 @@ -const challengeMarkers = /cloudflare|cf-chl|datadome|perimeterx|px-captcha|akamai|captcha|just a moment|verify you are human/i; +/** + * Headers that describe *this* response. Everything else is excluded on + * purpose: `content-security-policy`, `report-to` and `link` are allow-lists of + * third parties a page may load, so a CSP naming `cdnjs.cloudflare.com` or + * `google.com/recaptcha` proves nothing about who served the bytes (#264). + */ +const CHALLENGE_HEADERS = /^(?:server|cf-mitigated|cf-chl-[\w-]+|x-datadome[\w-]*|set-cookie)$/i; + +/** + * Markers that do not legitimately appear outside an actual challenge, so they + * decide on their own at any status — including the managed-challenge + * interstitial Cloudflare serves with a 200. + */ +const DECISIVE_HEADERS = /^(?:cf-mitigated|cf-chl-[\w-]+)$/i; +const DECISIVE_BODY_MARKERS = /cf-chl|cf-mitigated|just a moment|verify you are human|checking your browser|enable javascript and cookies/i; + +/** + * Markers that appear constantly on healthy pages: a CDN name in `server:`, + * or a reCAPTCHA widget embedded in an ordinary login form. These only + * corroborate a status that already looks like a block, never decide alone — + * that is the difference between a real block and the false positive in #283. + */ +const CORROBORATING_MARKERS = /cloudflare|datadome|perimeterx|px-captcha|akamai|captcha|__cf_bm/i; + +const BLOCKED_STATUSES = new Set([403, 429, 503]); export function isChallengeResponse(status: number, headers: Record, body: string): boolean { - const evidence = `${Object.entries(headers).map(([key, value]) => `${key}:${value}`).join('\n')}\n${body.slice(0, 20_000)}`; - return challengeMarkers.test(evidence) && (status === 403 || status === 429 || status === 503 || status === 200); + const headerEvidence = Object.entries(headers) + .filter(([key]) => CHALLENGE_HEADERS.test(key)) + .map(([key, value]) => `${key}:${value}`) + .join('\n'); + const bodyEvidence = body.slice(0, 20_000); + if (Object.entries(headers).some(([key, value]) => DECISIVE_HEADERS.test(key) && /challenge|1/i.test(value)) || DECISIVE_BODY_MARKERS.test(bodyEvidence)) return true; + const evidence = [ + headerEvidence, + bodyEvidence, + ].join('\n'); + return BLOCKED_STATUSES.has(status) && CORROBORATING_MARKERS.test(evidence); } export function isJavaScriptShell(body: string): boolean { diff --git a/src/fetch/client.test.ts b/src/fetch/client.test.ts index ad6844f4..cf5d8565 100644 --- a/src/fetch/client.test.ts +++ b/src/fetch/client.test.ts @@ -1,12 +1,14 @@ import { describe, expect, it, vi } from 'vitest'; +import { CliError } from '../errors.js'; import { webFetch } from './client.js'; function response(body: string, status = 200, headers: Record = { 'content-type': 'text/plain' }) { return new Response(body, { status, headers }); } +const safeProxy = (close = vi.fn().mockResolvedValue(undefined), policyError: () => Error | undefined = () => undefined) => ({ url: 'http://proxy', close, policyError }); describe('webFetch', () => { it('uses healthy plain responses without escalation', async () => { const plainFetch = vi.fn().mockResolvedValue(response('ok')); const createImpit = vi.fn(); - const result = await webFetch({ url: 'https://example.com', timeoutSeconds: 5, maxChars: 0, allowPrivate: false }, { plainFetch, createImpit, createSafeProxy: async () => ({ url: 'http://proxy', close: async () => {} }) }); + const result = await webFetch({ url: 'https://example.com', timeoutSeconds: 5, maxChars: 0, allowPrivate: false }, { plainFetch, createImpit, createSafeProxy: async () => safeProxy() }); expect(result).toMatchObject({ tier: 'plain', content: 'ok' }); expect(createImpit).not.toHaveBeenCalled(); }); @@ -14,7 +16,7 @@ describe('webFetch', () => { const first = { fetch: vi.fn().mockResolvedValue(response('challenge', 403, { server: 'cloudflare', 'content-type': 'text/plain' })) }; const second = { fetch: vi.fn().mockResolvedValue(response('ok')) }; const createImpit = vi.fn().mockReturnValueOnce(first).mockReturnValueOnce(second); - const result = await webFetch({ url: 'https://example.com', timeoutSeconds: 5, maxChars: 0, allowPrivate: false }, { plainFetch: vi.fn().mockResolvedValue(response('challenge', 403, { server: 'cloudflare', 'content-type': 'text/plain' })), createImpit, createSafeProxy: async () => ({ url: 'http://proxy', close: async () => {} }) }); + const result = await webFetch({ url: 'https://example.com', timeoutSeconds: 5, maxChars: 0, allowPrivate: false }, { plainFetch: vi.fn().mockResolvedValue(response('challenge', 403, { server: 'cloudflare', 'content-type': 'text/plain' })), createImpit, createSafeProxy: async () => safeProxy() }); expect(result).toMatchObject({ tier: 'impit', profile: 'firefox', content: 'ok' }); expect(createImpit).toHaveBeenNthCalledWith(1, expect.objectContaining({ browser: 'chrome' })); expect(createImpit).toHaveBeenNthCalledWith(2, expect.objectContaining({ browser: 'firefox' })); @@ -23,8 +25,8 @@ describe('webFetch', () => { const close = vi.fn().mockResolvedValue(undefined); await expect(webFetch({ url: 'https://example.com', timeoutSeconds: 5, maxChars: 0, allowPrivate: false }, { plainFetch: vi.fn().mockRejectedValue(new Error('boom')), - createImpit: vi.fn(), - createSafeProxy: async () => ({ url: 'http://proxy', close }), + createImpit: vi.fn(() => ({ fetch: vi.fn().mockRejectedValue(new Error('boom')) })), + createSafeProxy: async () => safeProxy(close), })).rejects.toThrow('boom'); expect(close).toHaveBeenCalledOnce(); }); @@ -32,8 +34,8 @@ describe('webFetch', () => { const abort = Object.assign(new Error('The operation was aborted'), { name: 'TimeoutError' }); await expect(webFetch({ url: 'https://example.com', timeoutSeconds: 5, maxChars: 0, allowPrivate: false }, { plainFetch: vi.fn().mockRejectedValue(abort), - createImpit: vi.fn(), - createSafeProxy: async () => ({ url: 'http://proxy', close: async () => {} }), + createImpit: vi.fn(() => ({ fetch: vi.fn().mockRejectedValue(Object.assign(new Error('connect ECONNREFUSED'), { code: 'ECONNREFUSED' })) })), + createSafeProxy: async () => safeProxy(), })).rejects.toMatchObject({ code: 'TIMEOUT', message: 'web fetch timed out after 5s' }); }); it('reports an impit-shaped deadline as a structured timeout', async () => { @@ -42,15 +44,89 @@ describe('webFetch', () => { const impitTimeout = new Error('error sending request for url (https://example.com/): operation timed out'); await expect(webFetch({ url: 'https://example.com', timeoutSeconds: 0.05, maxChars: 0, allowPrivate: false }, { plainFetch: vi.fn().mockImplementation(async () => { await new Promise(done => setTimeout(done, 80)); throw impitTimeout; }), - createImpit: vi.fn(), - createSafeProxy: async () => ({ url: 'http://proxy', close: async () => {} }), + createImpit: vi.fn(() => ({ fetch: vi.fn().mockRejectedValue(Object.assign(new Error('connect ECONNREFUSED'), { code: 'ECONNREFUSED' })) })), + createSafeProxy: async () => safeProxy(), })).rejects.toMatchObject({ code: 'TIMEOUT', message: 'web fetch timed out after 0.05s' }); }); + it('times out a body read and closes the proxy', async () => { + const close = vi.fn().mockResolvedValue(undefined); + const cancel = vi.fn(); + const hanging = { + status: 200, + headers: new Headers({ 'content-type': 'text/plain' }), + url: 'https://example.com', + body: new ReadableStream({ pull: () => new Promise(() => {}), cancel }), + }; + await expect(webFetch({ url: 'https://example.com', timeoutSeconds: 0.05, maxChars: 0, allowPrivate: false }, { + plainFetch: vi.fn().mockResolvedValue(hanging), + createImpit: vi.fn(), + createSafeProxy: async () => safeProxy(close), + })).rejects.toMatchObject({ code: 'TIMEOUT' }); + expect(cancel).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + }); it('does not relabel a failure that happened with budget left', async () => { await expect(webFetch({ url: 'https://example.com', timeoutSeconds: 30, maxChars: 0, allowPrivate: false }, { plainFetch: vi.fn().mockRejectedValue(Object.assign(new Error('connect ECONNREFUSED'), { code: 'ECONNREFUSED' })), - createImpit: vi.fn(), - createSafeProxy: async () => ({ url: 'http://proxy', close: async () => {} }), + createImpit: vi.fn(() => ({ fetch: vi.fn().mockRejectedValue(Object.assign(new Error('connect ECONNREFUSED'), { code: 'ECONNREFUSED' })) })), + createSafeProxy: async () => safeProxy(), })).rejects.toThrow('connect ECONNREFUSED'); }); }); + +describe('webFetch fixed non-browser ladder', () => { + const options = { url: 'https://example.com', timeoutSeconds: 5, maxChars: 0, allowPrivate: false }; + const challenge = () => response('Just a moment', 403, { server: 'cloudflare', 'content-type': 'text/plain' }); + + it.each([ + ['plain challenge reaches Chrome', [challenge()], [response('chrome')], ['chrome'], { tier: 'impit', profile: 'chrome' }], + ['two challenges reach Firefox', [challenge()], [challenge(), response('firefox')], ['chrome', 'firefox'], { tier: 'impit', profile: 'firefox' }], + ])('%s', async (_name, plainResponses, impitResponses, createdProfiles, expected) => { + const impits = impitResponses.map(value => ({ fetch: vi.fn().mockResolvedValue(value) })); + const createImpit = vi.fn().mockImplementation(() => impits.shift()); + const result = await webFetch(options, { plainFetch: vi.fn().mockResolvedValue(plainResponses[0]), createImpit, createSafeProxy: async () => safeProxy() }); + expect(createImpit.mock.calls.map(([value]) => value.browser)).toEqual(createdProfiles); + expect(result).toMatchObject(expected); + }); + + it('advances transport failures through Firefox', async () => { + const createdProfiles: string[] = []; + const createImpit = vi.fn(({ browser }) => { + createdProfiles.push(browser); + return { fetch: vi.fn().mockImplementation(() => browser === 'chrome' ? Promise.reject(new Error('TLS')) : Promise.resolve(response('firefox'))) }; + }); + const result = await webFetch(options, { plainFetch: vi.fn().mockRejectedValue(new Error('socket')), createImpit, createSafeProxy: async () => safeProxy() }); + expect(createdProfiles).toEqual(['chrome', 'firefox']); + expect(result).toMatchObject({ tier: 'impit', profile: 'firefox' }); + }); + + it('gives Chrome and Firefox decreasing positive timeouts from one deadline', async () => { + const createImpit = vi.fn((_options: { browser: 'chrome' | 'firefox'; proxyUrl: string; timeout: number }) => ({ fetch: vi.fn().mockImplementation(async () => { await new Promise(done => setTimeout(done, 5)); return challenge(); }) })); + await expect(webFetch({ ...options, timeoutSeconds: 1 }, { plainFetch: vi.fn().mockResolvedValue(challenge()), createImpit, createSafeProxy: async () => safeProxy() })).rejects.toMatchObject({ code: 'FETCH_BLOCKED' }); + const timeouts = createImpit.mock.calls.map(([value]) => value.timeout); + expect(timeouts).toHaveLength(2); + expect(timeouts[0]).toBeGreaterThan(timeouts[1]); + expect(timeouts[1]).toBeGreaterThan(0); + }); + + it('stops terminal errors and uses the explicit Session workflow', async () => { + const createImpit = vi.fn(); + await expect(webFetch(options, { plainFetch: vi.fn().mockResolvedValue(response('
')), createImpit, createSafeProxy: async () => safeProxy() })).rejects.toMatchObject({ code: 'FETCH_REQUIRES_BROWSER', hint: expect.stringContaining('session create') }); + expect(createImpit).not.toHaveBeenCalled(); + await expect(webFetch(options, { plainFetch: vi.fn().mockRejectedValue(new CliError('FETCH_BODY_TOO_LARGE', 'large')), createImpit, createSafeProxy: async () => safeProxy() })).rejects.toMatchObject({ code: 'FETCH_BODY_TOO_LARGE' }); + expect(createImpit).not.toHaveBeenCalled(); + }); + + it('stops for a proxy policy error and closes once', async () => { + const close = vi.fn().mockResolvedValue(undefined); + const createImpit = vi.fn(); + await expect(webFetch(options, { plainFetch: vi.fn().mockRejectedValue(new Error('proxy failure')), createImpit, createSafeProxy: async () => safeProxy(close, () => new Error('Unsafe fetch destination: 127.0.0.1')) })).rejects.toMatchObject({ code: 'FETCH_UNSAFE_ADDRESS' }); + expect(createImpit).not.toHaveBeenCalled(); + expect(close).toHaveBeenCalledOnce(); + }); + + it('returns blocked after three completed challenges with the explicit Session workflow', async () => { + const createImpit = vi.fn().mockReturnValueOnce({ fetch: vi.fn().mockResolvedValue(challenge()) }).mockReturnValueOnce({ fetch: vi.fn().mockResolvedValue(challenge()) }); + await expect(webFetch(options, { plainFetch: vi.fn().mockResolvedValue(challenge()), createImpit, createSafeProxy: async () => safeProxy() })).rejects.toMatchObject({ code: 'FETCH_BLOCKED', hint: expect.stringContaining('browser run --stdin') }); + }); +}); diff --git a/src/fetch/client.ts b/src/fetch/client.ts index 07d701af..92211d72 100644 --- a/src/fetch/client.ts +++ b/src/fetch/client.ts @@ -19,18 +19,27 @@ export interface WebFetchDependencies { createSafeProxy?: (options: { allowPrivate: boolean }) => Promise; } const MAX_BODY_BYTES = 10 * 1024 * 1024; +const BROWSER_WORKFLOW = 'Create a browser Session with `webcmd --profile work session create`, then navigate with `webcmd --profile work --session browser run --stdin`.'; function headersOf(response: ResponseLike): Record { return Object.fromEntries(response.headers.entries()); } -async function readBody(response: ResponseLike): Promise { +function beforeDeadline(promise: Promise, deadline: number, timeoutSeconds: number, cancel?: () => void): Promise { + const ms = deadline - Date.now(); + if (ms <= 0) { cancel?.(); return Promise.reject(new TimeoutError('web fetch', timeoutSeconds)); } + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { cancel?.(); reject(new TimeoutError('web fetch', timeoutSeconds)); }, ms); + promise.then(value => { clearTimeout(timer); resolve(value); }, error => { clearTimeout(timer); reject(error); }); + }); +} +async function readBody(response: ResponseLike, deadline: number, timeoutSeconds: number): Promise { if (!response.body && response.bytes) { - const bytes = await response.bytes(); + const bytes = await beforeDeadline(response.bytes(), deadline, timeoutSeconds); if (bytes.byteLength > MAX_BODY_BYTES) throw new CliError('FETCH_BODY_TOO_LARGE', 'Fetched body exceeds 10 MiB'); return new TextDecoder().decode(bytes); } const reader = response.body?.getReader(); if (!reader) return ''; const chunks: Uint8Array[] = []; let size = 0; - while (true) { const { done, value } = await reader.read(); if (done) break; size += value.byteLength; if (size > MAX_BODY_BYTES) { await reader.cancel(); throw new CliError('FETCH_BODY_TOO_LARGE', 'Fetched body exceeds 10 MiB'); } chunks.push(value); } + while (true) { const { done, value } = await beforeDeadline(reader.read(), deadline, timeoutSeconds, () => { void reader.cancel(); }); if (done) break; size += value.byteLength; if (size > MAX_BODY_BYTES) { await reader.cancel(); throw new CliError('FETCH_BODY_TOO_LARGE', 'Fetched body exceeds 10 MiB'); } chunks.push(value); } return new TextDecoder().decode(Buffer.concat(chunks)); } function truncate(content: string, limit: number): { content: string; truncated: boolean } { @@ -46,26 +55,35 @@ export async function webFetch(options: WebFetchOptions, dependencies: WebFetchD const plainFetch = dependencies.plainFetch ?? ((url, init) => fetch(url, init)); const createImpit = dependencies.createImpit ?? (impitOptions => new Impit(impitOptions)); try { - let response = await plainFetch(options.url, { redirect: 'manual', dispatcher: new ProxyAgent(proxy.url), signal: AbortSignal.timeout(remaining()) }); - let body = await readBody(response); - let tier: WebFetchResult['tier'] = 'plain'; let profile: WebFetchResult['profile']; - if (isJavaScriptShell(body)) throw new CliError('FETCH_REQUIRES_BROWSER', 'This page requires browser rendering.', 'Use webcmd web fetch-browser for this URL.'); - if (isChallengeResponse(response.status, headersOf(response), body)) { - // ponytail: impit's timeout covers the request, not the body stream, so a - // trickling escalation body can outlive the budget. Race readBody against - // the deadline if that shows up in practice. - for (const browser of ['chrome', 'firefox'] as const) { - const impit = createImpit({ browser, proxyUrl: proxy.url, timeout: remaining() }); - response = await impit.fetch(options.url, { redirect: 'manual', timeout: remaining() }); - body = await readBody(response); tier = 'impit'; profile = browser; - if (isJavaScriptShell(body)) throw new CliError('FETCH_REQUIRES_BROWSER', 'This page requires browser rendering.', 'Use webcmd web fetch-browser for this URL.'); - if (!isChallengeResponse(response.status, headersOf(response), body)) break; + const ladder = [undefined, 'chrome', 'firefox'] as const; + let lastTransport: unknown; + for (const browser of ladder) { + try { + const timeout = remaining(); + const response = browser + ? await createImpit({ browser, proxyUrl: proxy.url, timeout }).fetch(options.url, { redirect: 'manual', timeout: remaining() }) + : await plainFetch(options.url, { redirect: 'manual', dispatcher: new ProxyAgent(proxy.url), signal: AbortSignal.timeout(timeout) }); + const policyError = proxy.policyError(); + if (policyError) throw new CliError('FETCH_UNSAFE_ADDRESS', policyError.message); + const body = await readBody(response, deadline, options.timeoutSeconds); + if (proxy.policyError()) throw new CliError('FETCH_UNSAFE_ADDRESS', proxy.policyError()!.message); + if (isJavaScriptShell(body)) throw new CliError('FETCH_REQUIRES_BROWSER', 'This page requires browser rendering.', BROWSER_WORKFLOW); + if (isChallengeResponse(response.status, headersOf(response), body)) { + if (browser === 'firefox') throw new CliError('FETCH_BLOCKED', 'The site blocked non-browser fetches.', BROWSER_WORKFLOW); + continue; + } + const extracted = extractFetchedContent({ body, contentType: response.headers.get('content-type') ?? '', url: options.url }); + const clipped = truncate(extracted.content, options.maxChars); + return { status: response.status, requestedUrl: options.url, finalUrl: response.url || options.url, contentType: response.headers.get('content-type') ?? '', tier: browser ? 'impit' : 'plain', ...(browser && { profile: browser }), title: extracted.title, extractionSource: extracted.source, truncated: clipped.truncated, content: clipped.content }; + } catch (error) { + const policyError = proxy.policyError(); + if (policyError) throw new CliError('FETCH_UNSAFE_ADDRESS', policyError.message); + const mapped = asFetchError(error, options.timeoutSeconds, deadline); + if (mapped instanceof CliError) throw mapped; + lastTransport = mapped; } - if (isChallengeResponse(response.status, headersOf(response), body)) throw new CliError('FETCH_BLOCKED', 'The site blocked non-browser fetches.', 'Use webcmd web fetch-browser for this URL.'); } - const extracted = extractFetchedContent({ body, contentType: response.headers.get('content-type') ?? '', url: options.url }); - const clipped = truncate(extracted.content, options.maxChars); - return { status: response.status, requestedUrl: options.url, finalUrl: response.url || options.url, contentType: response.headers.get('content-type') ?? '', tier, ...(profile && { profile }), title: extracted.title, extractionSource: extracted.source, truncated: clipped.truncated, content: clipped.content }; + throw lastTransport; } catch (error) { throw asFetchError(error, options.timeoutSeconds, deadline); } finally { await proxy.close(); } diff --git a/src/fetch/command.test.ts b/src/fetch/command.test.ts index 5b9823d2..d50da849 100644 --- a/src/fetch/command.test.ts +++ b/src/fetch/command.test.ts @@ -1,13 +1,111 @@ -import { describe, expect, it, vi } from 'vitest'; -import { formatWebFetchMarkdown, runClientOwnedWebFetch } from './command.js'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { Command } from 'commander'; + +const { mockWebFetch, mockRenderOutput } = vi.hoisted(() => ({ + mockWebFetch: vi.fn(), + mockRenderOutput: vi.fn(), +})); + +vi.mock('./client.js', () => ({ webFetch: mockWebFetch })); +vi.mock('../execution.js', () => { + throw new Error('web fetch must not import the generic executor'); +}); +vi.mock('../output.js', async () => ({ + ...(await vi.importActual('../output.js')), + render: mockRenderOutput, +})); + +import { registerCommandToProgram } from '../commanderAdapter.js'; +import { formatWebFetchMarkdown, runWebFetchCommand, webFetchCommand } from './command.js'; + +const plainResult = { status: 200, requestedUrl: 'https://example.com', finalUrl: 'https://example.com', contentType: 'text/plain', tier: 'plain' as const, title: 'Example', extractionSource: 'raw' as const, truncated: false, content: 'ok' }; + +function program(): Command { + const root = new Command().exitOverride(); + registerCommandToProgram(root.command('web'), webFetchCommand); + return root; +} describe('web fetch command', () => { - it('renders fetch metadata before content', () => { - expect(formatWebFetchMarkdown({ status: 200, requestedUrl: 'https://a', finalUrl: 'https://b', contentType: 'text/plain', tier: 'plain', title: 'T', extractionSource: 'raw', truncated: false, content: 'body' })).toContain('Source: https://a'); + beforeEach(() => { + mockWebFetch.mockReset().mockResolvedValue(plainResult); + mockRenderOutput.mockReset(); + process.exitCode = undefined; }); - it('runs the client-owned command without Cloud routing', async () => { - const webFetch = vi.fn().mockResolvedValue({ status: 200, requestedUrl: 'https://a', finalUrl: 'https://a', contentType: 'text/plain', tier: 'plain', title: '', extractionSource: 'raw', truncated: false, content: 'ok' }); - await runClientOwnedWebFetch(['web', 'fetch', '--url', 'https://a'], { webFetch, stdout: { write: vi.fn() } as never }); - expect(webFetch).toHaveBeenCalledOnce(); + + it('is the client-owned, non-browser core command', () => { + expect(webFetchCommand).toMatchObject({ site: 'web', name: 'fetch', browser: false, clientOwned: true, defaultFormat: 'md' }); + }); + + it('accepts hosted root options without importing execution', async () => { + await runWebFetchCommand(['--profile', 'work', '--workspace', 'test', 'web', 'fetch', '--url', 'https://example.com']); + + expect(mockWebFetch).toHaveBeenCalledWith({ url: 'https://example.com', timeoutSeconds: 30, maxChars: 50_000, allowPrivate: false }); + }); + + it('uses Commander coercion for canonical fetch options', async () => { + await program().parseAsync(['web', 'fetch', '--url=https://example.com', '--timeout=9', '--max-chars=1200', '--allow-private=false', '--format=json'], { from: 'user' }); + + expect(mockWebFetch).toHaveBeenCalledWith({ url: 'https://example.com', timeoutSeconds: 9, maxChars: 1200, allowPrivate: false }); + }); + + it('shows help without requiring a URL', async () => { + await expect(program().parseAsync(['web', 'fetch', '--help'], { from: 'user' })).rejects.toMatchObject({ code: 'commander.helpDisplayed' }); + expect(mockWebFetch).not.toHaveBeenCalled(); + }); + + it('uses structured JSON help without requiring a URL', async () => { + const originalArgv = process.argv; + const output: string[] = []; + const write = vi.spyOn(process.stdout, 'write').mockImplementation((chunk: string | Uint8Array) => { + output.push(String(chunk)); + return true; + }); + process.argv = ['node', 'webcmd', 'web', 'fetch', '--help', '-f', 'json']; + try { + await expect(program().parseAsync(['web', 'fetch', '--help', '-f', 'json'], { from: 'user' })).rejects.toMatchObject({ code: 'commander.helpDisplayed' }); + expect(JSON.parse(output.join(''))).toMatchObject({ site: 'web', name: 'fetch' }); + } finally { + process.argv = originalArgv; + write.mockRestore(); + } + }); + + it('accepts both output-format spellings', async () => { + for (const args of [['--format', 'json'], ['--format=json']]) { + await program().parseAsync(['web', 'fetch', '--url', 'https://example.com', ...args], { from: 'user' }); + } + expect(mockRenderOutput).toHaveBeenCalledTimes(2); + expect(mockRenderOutput.mock.calls.map(call => call[1].fmt)).toEqual(['json', 'json']); + }); + + it('renders fetched documents as markdown but preserves structured JSON', async () => { + await program().parseAsync(['web', 'fetch', '--url', 'https://example.com', '-f', 'md'], { from: 'user' }); + await program().parseAsync(['web', 'fetch', '--url', 'https://example.com', '-f', 'json'], { from: 'user' }); + const markdown = mockRenderOutput.mock.calls[0][1].markdown as (data: unknown) => string | undefined; + expect(markdown(plainResult)).toBe(formatWebFetchMarkdown(plainResult)); + expect(mockRenderOutput.mock.calls[1][1]).toMatchObject({ fmt: 'json' }); + }); + + it.each([ + ['--timeout', 'nope'], ['--timeout', '-1'], ['--max-chars', '-1'], ['--url', 'ftp://example.com'], + ])('rejects invalid fetch arguments (%s %s)', async (...args) => { + const output: string[] = []; + const write = vi.spyOn(process.stderr, 'write').mockImplementation((chunk: string | Uint8Array) => { + output.push(String(chunk)); + return true; + }); + try { + await program().parseAsync(['web', 'fetch', '--url', 'https://example.com', ...args], { from: 'user' }); + expect(mockWebFetch).not.toHaveBeenCalled(); + expect(output.join('')).toContain('code: ARGUMENT'); + expect(process.exitCode).toBe(2); + } finally { + write.mockRestore(); + } + }); + + it.each([['--browser'], ['--wait', '1'], ['--unknown']])('rejects removed and unknown options (%s)', async (...args) => { + await expect(program().parseAsync(['web', 'fetch', '--url', 'https://example.com', ...args], { from: 'user' })).rejects.toMatchObject({ code: 'commander.unknownOption' }); }); }); diff --git a/src/fetch/command.ts b/src/fetch/command.ts index 103e8487..dbb0e808 100644 --- a/src/fetch/command.ts +++ b/src/fetch/command.ts @@ -1,37 +1,58 @@ -import { cli, Strategy } from '../registry.js'; +import { Command } from 'commander'; +import { cli, Strategy, type CommandArgs } from '../registry.js'; +import { registerCommandToProgram } from '../commanderAdapter.js'; +import { configureRootCommandSurface } from '../root-command-surface.js'; import { ArgumentError } from '../errors.js'; -import { webFetch, type WebFetchOptions, type WebFetchResult } from './client.js'; +import type { WebFetchOptions, WebFetchResult } from './client.js'; export const webFetchCommand = cli({ site: 'web', name: 'fetch', access: 'read', strategy: Strategy.PUBLIC, browser: false, - description: 'Fetch a URL locally without launching a browser', defaultFormat: 'md', + clientOwned: true, + description: 'Fetch a URL with local HTTP clients', defaultFormat: 'md', + renderMarkdown: data => (isWebFetchResult(data) ? formatWebFetchMarkdown(data) : undefined), args: [ - { name: 'url', type: 'string', required: true }, - { name: 'timeout', type: 'int', default: 30 }, - { name: 'max-chars', type: 'int', default: 50000 }, - { name: 'allow-private', type: 'boolean', default: false }, + { name: 'url', type: 'string', required: true, help: 'HTTP or HTTPS URL to fetch' }, + { name: 'timeout', type: 'int', default: 30, help: 'Total fetch budget in seconds' }, + { name: 'max-chars', type: 'int', default: 50_000, help: 'Maximum extracted characters; 0 disables truncation' }, + { name: 'allow-private', type: 'boolean', default: false, help: 'Allow private and loopback destinations' }, ], - func: async kwargs => webFetch({ url: String(kwargs.url), timeoutSeconds: Number(kwargs.timeout ?? 30), maxChars: Number(kwargs['max-chars'] ?? 50000), allowPrivate: kwargs['allow-private'] === true }), + validateArgs: validateWebFetchArgs, + func: async (kwargs) => { + const { webFetch } = await import('./client.js'); + return webFetch(clientOptionsFromKwargs(kwargs)); + }, }); -export function formatWebFetchMarkdown(result: WebFetchResult): string { - return [`# ${result.title || 'Fetched content'}`, '', `Source: ${result.requestedUrl}`, `Final URL: ${result.finalUrl}`, `Content type: ${result.contentType || 'unknown'}`, `Extraction: ${result.extractionSource}`, '', result.content].join('\n'); +/** Run only the client-owned fetch command without loading the main CLI. */ +export async function runWebFetchCommand(argv: string[]): Promise { + const program = configureRootCommandSurface(new Command('webcmd')) + .option('--workspace ', 'Hosted workspace id/slug for the request'); + registerCommandToProgram(program.command('web'), webFetchCommand); + await program.parseAsync(argv, { from: 'user' }); +} + +function clientOptionsFromKwargs(kwargs: CommandArgs): WebFetchOptions { + return { + url: String(kwargs.url), + timeoutSeconds: Number(kwargs.timeout ?? 30), + maxChars: Number(kwargs['max-chars'] ?? 50000), + allowPrivate: kwargs['allow-private'] === true, + }; } -function clientOptions(argv: readonly string[]): WebFetchOptions { - const values: Record = {}; - for (let index = 2; index < argv.length; index++) { - const arg = argv[index]!; - if (!arg.startsWith('--')) continue; - const name = arg.slice(2); const value = argv[index + 1]; - if (value && !value.startsWith('--')) { values[name] = value; index++; } else values[name] = true; +function validateWebFetchArgs(kwargs: CommandArgs): void { + let url: URL; + try { url = new URL(String(kwargs.url)); } catch { throw new ArgumentError('--url must be an http or https URL'); } + if (url.protocol !== 'http:' && url.protocol !== 'https:') throw new ArgumentError('--url must be an http or https URL'); + for (const name of ['timeout', 'max-chars']) { + if (!Number.isInteger(kwargs[name]) || kwargs[name] < 0) throw new ArgumentError(`--${name} must be a non-negative integer`); } - if (typeof values.url !== 'string' || !/^https?:\/\//i.test(values.url)) throw new ArgumentError('--url must be an http or https URL'); - const int = (name: string, fallback: number) => { const value = values[name]; const number = value === undefined ? fallback : Number(value); if (!Number.isInteger(number) || number < 0) throw new ArgumentError(`--${name} must be a non-negative integer`); return number; }; - return { url: values.url, timeoutSeconds: int('timeout', 30), maxChars: int('max-chars', 50000), allowPrivate: values['allow-private'] === true || values['allow-private'] === 'true' }; } -export async function runClientOwnedWebFetch(argv: readonly string[], dependencies: { webFetch?: typeof webFetch; stdout?: NodeJS.WritableStream } = {}): Promise { - const result = await (dependencies.webFetch ?? webFetch)(clientOptions(argv)); - (dependencies.stdout ?? process.stdout).write(`${formatWebFetchMarkdown(result)}\n`); +function isWebFetchResult(data: unknown): data is WebFetchResult { + return typeof data === 'object' && data !== null && 'requestedUrl' in data && 'content' in data; +} + +export function formatWebFetchMarkdown(result: WebFetchResult): string { + return [`# ${result.title || 'Fetched content'}`, '', `Source: ${result.requestedUrl}`, `Final URL: ${result.finalUrl}`, `Content type: ${result.contentType || 'unknown'}`, `Extraction: ${result.extractionSource}`, '', result.content].join('\n'); } diff --git a/src/fetch/extract.test.ts b/src/fetch/extract.test.ts index 945abf7d..404b9444 100644 --- a/src/fetch/extract.test.ts +++ b/src/fetch/extract.test.ts @@ -25,4 +25,14 @@ describe('extractFetchedContent', () => { } throw new Error('expected unsupported content type'); }); + + it('does not recommend browser rendering for binary downloads', () => { + try { + extractFetchedContent({ url: 'https://example.com/a', contentType: 'application/octet-stream', body: '' }); + } catch (error) { + expect(error).toMatchObject({ code: 'FETCH_UNSUPPORTED_CONTENT_TYPE', hint: undefined }); + return; + } + throw new Error('expected unsupported content type'); + }); }); diff --git a/src/fetch/extract.ts b/src/fetch/extract.ts index 83e9b3d2..6ebd2c29 100644 --- a/src/fetch/extract.ts +++ b/src/fetch/extract.ts @@ -32,10 +32,13 @@ export function extractFetchedContent(input: ExtractFetchedContentInput): Extrac return { title: '', content: input.body, source: 'raw' }; } if (!['text/html', 'application/xhtml+xml', ''].includes(contentType)) { + const hint = contentType === 'application/pdf' + ? 'Create a browser Session with `webcmd --profile work session create`, then navigate with `webcmd --profile work --session browser run --stdin`.' + : undefined; throw new CliError( 'FETCH_UNSUPPORTED_CONTENT_TYPE', `Unsupported content type: ${contentType || 'unknown'}`, - 'Use webcmd web fetch-browser for content that requires browser rendering.', + hint, ); } diff --git a/src/fetch/safe-proxy.test.ts b/src/fetch/safe-proxy.test.ts index 5b8e51db..9dfe8551 100644 --- a/src/fetch/safe-proxy.test.ts +++ b/src/fetch/safe-proxy.test.ts @@ -1,7 +1,14 @@ import * as net from 'node:net'; +import { spawn } from 'node:child_process'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import * as path from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; import { describe, expect, it } from 'vitest'; import { createSafeProxy, isSafeAddress } from './safe-proxy.js'; +const moduleDir = path.dirname(fileURLToPath(import.meta.url)); + describe('isSafeAddress', () => { it.each(['127.0.0.1', '10.0.0.1', '172.16.0.1', '192.168.1.1', '169.254.169.254', '0.0.0.0', '::1', '::', 'fe80::1', '::ffff:127.0.0.1'])('rejects private address %s', address => { expect(isSafeAddress(address)).toBe(false); @@ -10,6 +17,69 @@ describe('isSafeAddress', () => { }); describe('createSafeProxy close', () => { + it('does not crash the process when a tunnel client resets', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'webcmd-safe-proxy-rst-')); + const scriptPath = path.join(dir, 'repro.mjs'); + const safeProxyUrl = pathToFileURL(path.join(moduleDir, 'safe-proxy.ts')).href; + await writeFile(scriptPath, [ + "import * as net from 'node:net';", + `import { createSafeProxy } from ${JSON.stringify(safeProxyUrl)};`, + "const upstream = net.createServer(socket => socket.resume());", + "await new Promise(done => upstream.listen(0, '127.0.0.1', done));", + 'const proxy = await createSafeProxy({ allowPrivate: true });', + 'const client = net.connect({ host: "127.0.0.1", port: Number(new URL(proxy.url).port) });', + 'await new Promise((resolve, reject) => {', + ' client.once("error", reject);', + ' client.once("data", resolve);', + ' client.write(`CONNECT 127.0.0.1:${upstream.address().port} HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\n\\r\\n`);', + '});', + 'client.resetAndDestroy();', + 'await new Promise(done => setTimeout(done, 100));', + 'await proxy.close();', + 'await new Promise(done => upstream.close(done));', + ].join('\n')); + + try { + const result = await new Promise<{ status: number | null; stderr: string }>((resolve, reject) => { + const child = spawn(process.execPath, ['--import', 'tsx', scriptPath], { stdio: ['ignore', 'ignore', 'pipe'] }); + const stderr: Buffer[] = []; + child.stderr.on('data', chunk => stderr.push(Buffer.from(chunk))); + child.once('error', reject); + child.once('close', status => resolve({ status, stderr: Buffer.concat(stderr).toString('utf8') })); + }); + expect(result.stderr).not.toContain('ECONNRESET'); + expect(result.status).toBe(0); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }, 15_000); + + it('destroys the upstream tunnel when the client leg resets', async () => { + let upstreamSocket: net.Socket | undefined; + const upstream = net.createServer(socket => { upstreamSocket = socket; socket.resume(); }); + await new Promise(done => upstream.listen(0, '127.0.0.1', () => done())); + const upstreamPort = (upstream.address() as net.AddressInfo).port; + const proxy = await createSafeProxy({ allowPrivate: true }); + + const client = net.connect({ host: '127.0.0.1', port: Number(new URL(proxy.url).port) }); + client.on('error', () => {}); + await new Promise(done => { + client.once('data', () => done()); + client.write(`CONNECT 127.0.0.1:${upstreamPort} HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n`); + }); + expect(upstreamSocket).toBeDefined(); + + const upstreamClosed = new Promise(resolve => { + upstreamSocket!.once('close', () => resolve(true)); + setTimeout(() => resolve(false), 250); + }); + client.resetAndDestroy(); + + expect(await upstreamClosed).toBe(true); + await proxy.close(); + await new Promise(done => upstream.close(() => done())); + }); + it('does not wait for an idle CONNECT tunnel to drain', async () => { // Stands in for the upstream host: accepts and then never says anything, // exactly like the keep-alive tunnels impit leaves behind. @@ -72,3 +142,25 @@ describe('createSafeProxy close', () => { await new Promise(done => upstream.close(() => done())); }); }); + +describe('createSafeProxy policy errors', () => { + it('records the first rejected private destination', async () => { + const proxy = await createSafeProxy({ allowPrivate: false }); + const client = net.connect({ host: '127.0.0.1', port: Number(new URL(proxy.url).port) }); + const reply = await new Promise(done => { + let data = ''; + client.on('data', chunk => { data += chunk; }); + client.on('end', () => done(data)); + client.write('CONNECT 127.0.0.1:443 HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n'); + }); + expect(reply).toContain('403 Forbidden'); + expect(proxy.policyError()?.message).toContain('Unsafe fetch destination'); + await proxy.close(); + }); + + it('leaves the policy slot empty when private addresses are allowed', async () => { + const proxy = await createSafeProxy({ allowPrivate: true }); + expect(proxy.policyError()).toBeUndefined(); + await proxy.close(); + }); +}); diff --git a/src/fetch/safe-proxy.ts b/src/fetch/safe-proxy.ts index 2d82527f..0ae3e0ed 100644 --- a/src/fetch/safe-proxy.ts +++ b/src/fetch/safe-proxy.ts @@ -3,7 +3,7 @@ import * as http from 'node:http'; import * as net from 'node:net'; import type { Duplex } from 'node:stream'; -export interface SafeProxy { url: string; close(): Promise; } +export interface SafeProxy { url: string; close(): Promise; policyError(): Error | undefined; } export interface SafeProxyOptions { allowPrivate?: boolean; lookup?: typeof dnsLookup; } export function isSafeAddress(address: string): boolean { @@ -49,6 +49,7 @@ export async function createSafeProxy(options: SafeProxyOptions = {}): Promise(socket: T): T => { if (closing) { socket.destroy(); return socket; } sockets.add(socket); @@ -58,31 +59,44 @@ export async function createSafeProxy(options: SafeProxyOptions = {}): Promise { + let upstream: http.ClientRequest | undefined; + request.on('error', () => { upstream?.destroy(); response.destroy(); }); + response.on('error', () => { upstream?.destroy(); request.destroy(); }); try { const target = new URL(request.url ?? ''); const address = await resolve(target.hostname, lookup, allowPrivate); if (closing) { response.destroy(); return; } - const upstream = http.request({ host: address, port: Number(target.port) || 80, method: request.method, path: `${target.pathname}${target.search}`, headers: { ...request.headers, host: target.host } }, upstreamResponse => { + upstream = http.request({ host: address, port: Number(target.port) || 80, method: request.method, path: `${target.pathname}${target.search}`, headers: { ...request.headers, host: target.host } }, upstreamResponse => { response.writeHead(upstreamResponse.statusCode ?? 502, upstreamResponse.headers); upstreamResponse.pipe(response); }); upstream.on('socket', track); upstream.on('error', error => response.destroy(error)); request.pipe(upstream); - } catch (error) { response.writeHead(403).end(error instanceof Error ? error.message : 'Unsafe fetch destination'); } + } catch (error) { + if (error instanceof Error && error.message.startsWith('Unsafe fetch destination:')) firstPolicyError ??= error; + response.writeHead(403).end(error instanceof Error ? error.message : 'Unsafe fetch destination'); + } }); server.on('connection', track); server.on('connect', async (request, client, head) => { track(client); + let upstream: net.Socket | undefined; + client.on('error', () => upstream?.destroy()); + client.on('close', () => upstream?.destroy()); try { const [host, portText] = (request.url ?? '').replace(/^\[/, '').replace(']', '').split(':'); if (!host) throw new Error('Invalid CONNECT target'); const address = await resolve(host, lookup, allowPrivate); if (closing) { client.destroy(); return; } - const upstream = track(net.connect({ host: address, port: Number(portText) || 443 })); - upstream.once('connect', () => { client.write('HTTP/1.1 200 Connection Established\r\n\r\n'); if (head.length) upstream.write(head); upstream.pipe(client); client.pipe(upstream); }); - upstream.once('error', error => client.destroy(error)); - } catch (error) { client.end(`HTTP/1.1 403 Forbidden\r\n\r\n${error instanceof Error ? error.message : ''}`); } + const tunnel = track(net.connect({ host: address, port: Number(portText) || 443 })); + upstream = tunnel; + tunnel.once('connect', () => { client.write('HTTP/1.1 200 Connection Established\r\n\r\n'); if (head.length) tunnel.write(head); tunnel.pipe(client); client.pipe(tunnel); }); + tunnel.on('error', error => client.destroy(error)); + } catch (error) { + if (error instanceof Error && error.message.startsWith('Unsafe fetch destination:')) firstPolicyError ??= error; + client.end(`HTTP/1.1 403 Forbidden\r\n\r\n${error instanceof Error ? error.message : ''}`); + } }); await new Promise((resolveListen, reject) => { server.once('error', reject); server.listen(0, '127.0.0.1', () => resolveListen()); }); const address = server.address(); @@ -90,6 +104,7 @@ export async function createSafeProxy(options: SafeProxyOptions = {}): Promise | undefined; return { url: `http://127.0.0.1:${address.port}`, + policyError: () => firstPolicyError, // Idempotent: a second close() awaits the first rather than asking an // already-stopped server to close again. close: () => (closed ??= new Promise((resolveClose, reject) => { diff --git a/src/hosted/availability.test.ts b/src/hosted/availability.test.ts index df4ce587..e46dc9e7 100644 --- a/src/hosted/availability.test.ts +++ b/src/hosted/availability.test.ts @@ -165,6 +165,11 @@ function exceptionDiff(actual: readonly string[], expected: readonly string[]) { } describe('hosted availability', () => { + it('keeps client-owned commands local before strategy and domain classification', () => { + expect(deriveHostedAvailability({ clientOwned: true, strategy: 'public', domain: 'example.com' })) + .toEqual({ mode: 'local-only', reason: 'client-owned' }); + }); + it('derives decisions only from normalized strategy and domain metadata', () => { expect(deriveHostedAvailability({ strategy: 'local', domain: 'localhost' })) .toEqual({ mode: 'local-only', reason: 'local-tool' }); diff --git a/src/hosted/availability.ts b/src/hosted/availability.ts index 3bcdcce0..43ae8f31 100644 --- a/src/hosted/availability.ts +++ b/src/hosted/availability.ts @@ -3,14 +3,16 @@ import { Strategy } from '../registry.js'; export type HostedAvailability = | { mode: 'hosted' } - | { mode: 'local-only'; reason: 'desktop-app' | 'local-tool' | 'browser-bind' }; + | { mode: 'local-only'; reason: 'client-owned' | 'desktop-app' | 'local-tool' | 'browser-bind' }; export interface HostedAvailabilityMetadata { + clientOwned?: boolean; strategy?: Strategy | string; domain?: string; } export function deriveHostedAvailability(command: HostedAvailabilityMetadata): HostedAvailability { + if (command.clientOwned) return { mode: 'local-only', reason: 'client-owned' }; if (String(command.strategy).toLowerCase() === Strategy.LOCAL) { return { mode: 'local-only', reason: 'local-tool' }; } diff --git a/src/hosted/contract.test.ts b/src/hosted/contract.test.ts index 94bc60a2..43843bb0 100644 --- a/src/hosted/contract.test.ts +++ b/src/hosted/contract.test.ts @@ -349,6 +349,22 @@ describe('buildHostedContract', () => { expect(command.keywords).not.toBe(keywords); }); + it('marks client-owned commands local-only while public commands remain hosted', () => { + const contract = buildHostedContract([ + { ...commands[0], name: 'fetch', clientOwned: true }, + { ...commands[0], aliases: undefined }, + ], [], '1.0.0'); + + expect(contract.commands.find(command => command.command === 'web/fetch')).toMatchObject({ + sessionPolicy: 'local-only', + availability: { mode: 'local-only', reason: 'client-owned' }, + }); + expect(contract.commands.find(command => command.command === 'web/profile')).toMatchObject({ + sessionPolicy: 'create-or-reuse', + availability: { mode: 'hosted' }, + }); + }); + it('rejects incomplete file and browser session metadata', () => { const missingDirection = { ...commands[2], diff --git a/src/hosted/contract.ts b/src/hosted/contract.ts index 3d527af5..a5b5d0e2 100644 --- a/src/hosted/contract.ts +++ b/src/hosted/contract.ts @@ -105,6 +105,7 @@ export interface HostedContract { } export interface HostedContractCommandInput { + clientOwned?: boolean; site: string; name: string; aliases?: string[]; diff --git a/src/hosted/main-lifecycle.test.ts b/src/hosted/main-lifecycle.test.ts index b5abbb77..de2f3a97 100644 --- a/src/hosted/main-lifecycle.test.ts +++ b/src/hosted/main-lifecycle.test.ts @@ -160,6 +160,35 @@ describe('hosted CLI process lifecycle', () => { expect(fixture.requests).toEqual([]); await expect(readFile(fixture.discoverySentinel, 'utf8')).rejects.toMatchObject({ code: 'ENOENT' }); }, 20_000); + + it('runs client-owned web fetch locally in hosted and local modes', async () => { + const fixture = await createHostedFixture('success'); + const article = createServer((_request, response) => { + response.writeHead(200, { 'content-type': 'text/html' }); + response.end('Fixture article
fixture content
'); + }); + servers.push(article); + await new Promise((resolve, reject) => { + article.once('error', reject); + article.listen(0, '127.0.0.1', resolve); + }); + const address = article.address(); + if (!address || typeof address === 'string') throw new Error('Expected TCP article fixture address'); + const args = ['web', 'fetch', '--url', `http://127.0.0.1:${address.port}/article`, '--allow-private', 'true', '-f', 'json']; + + const hosted = await runCli(args, fixture.env); + expect(hosted.status).toBe(0); + expect(JSON.parse(hosted.stdout).content).toContain('fixture content'); + expect(fixture.requests).toEqual([]); + await expect(readFile(fixture.discoverySentinel, 'utf8')).rejects.toMatchObject({ code: 'ENOENT' }); + + await writeFile(path.join(fixture.root, 'config', 'config.json'), '{"mode":"local"}\n'); + const local = await runCli(args, fixture.env); + expect(local.status).toBe(0); + expect(JSON.parse(local.stdout)).toMatchObject({ content: expect.stringContaining('fixture content') }); + expect(fixture.requests).toEqual([]); + await expect(readFile(fixture.discoverySentinel, 'utf8')).rejects.toMatchObject({ code: 'ENOENT' }); + }, 20_000); }); async function createHostedFixture(outcome: 'success' | 'failure'): Promise<{ diff --git a/src/hosted/manifest.test.ts b/src/hosted/manifest.test.ts index ab80c9c7..c1c642a2 100644 --- a/src/hosted/manifest.test.ts +++ b/src/hosted/manifest.test.ts @@ -16,6 +16,7 @@ import { renderHostedCommandHelp, renderHostedSiteHelp, siteNames, + withClientOwnedCommands, } from './manifest.js'; import type { HostedManifest } from './types.js'; import { makeHostedConfig } from './config.js'; @@ -87,6 +88,30 @@ function sink(): { stream: Writable; text: () => string } { } describe('hosted manifest helpers', () => { + it('merges the authoritative client-owned web fetch entry exactly once', () => { + const merged = withClientOwnedCommands({ + ...manifest, + commands: [ + ...manifest.commands, + { ...manifest.commands[0]!, site: 'web', name: 'fetch', command: 'web/fetch' }, + ], + }); + + expect(merged.commands.filter(command => command.command === 'web/fetch')).toEqual([ + expect.objectContaining({ + site: 'web', + name: 'fetch', + clientOwned: true, + args: expect.arrayContaining([ + expect.objectContaining({ name: 'url' }), + expect.objectContaining({ name: 'timeout' }), + expect.objectContaining({ name: 'max-chars' }), + expect.objectContaining({ name: 'allow-private' }), + ]), + }), + ]); + }); + it('filters LOCAL commands from hosted list rows', () => { expect(hostedListRows(manifest, true).map((row) => row.command)).toEqual(['github/whoami']); }); @@ -192,7 +217,7 @@ describe('hosted manifest helpers', () => { )); }); - it('uses only executable hosted root capabilities as root completion candidates', async () => { + it('includes client-owned web fetch in hosted discovery surfaces', async () => { const stdout = sink(); await runHostedCli(['--get-completions', '--cursor', '1'], { config: makeHostedConfig({ apiBaseUrl: 'https://api.example.com', apiKey: 'key' }), @@ -200,6 +225,32 @@ describe('hosted manifest helpers', () => { fetchImpl: async () => new Response(JSON.stringify({ ok: true, manifest }), { status: 200 }), }); - expect(stdout.text().trim().split('\n')).toEqual(['browser', 'completion', 'github', 'list', 'profile', 'setup']); + expect(stdout.text().trim().split('\n')).toEqual(['browser', 'completion', 'github', 'list', 'profile', 'setup', 'web']); + + const siteHelp = sink(); + await runHostedCli(['web', '--help'], { + config: makeHostedConfig({ apiBaseUrl: 'https://api.example.com', apiKey: 'key' }), + stdout: siteHelp.stream, + fetchImpl: async () => new Response(JSON.stringify({ ok: true, manifest }), { status: 200 }), + }); + expect(siteHelp.text()).toContain('fetch'); + + const completion = sink(); + await runHostedCli(['--get-completions', '--cursor', '2', 'web'], { + config: makeHostedConfig({ apiBaseUrl: 'https://api.example.com', apiKey: 'key' }), + stdout: completion.stream, + fetchImpl: async () => new Response(JSON.stringify({ ok: true, manifest }), { status: 200 }), + }); + expect(completion.text().trim()).toBe('fetch'); + + const list = sink(); + await runHostedCli(['list', '-f', 'json'], { + config: makeHostedConfig({ apiBaseUrl: 'https://api.example.com', apiKey: 'key' }), + stdout: list.stream, + fetchImpl: async () => new Response(JSON.stringify({ ok: true, manifest }), { status: 200 }), + }); + expect(JSON.parse(list.text())).toEqual(expect.arrayContaining([ + expect.objectContaining({ command: 'web/fetch', clientOwned: true }), + ])); }); }); diff --git a/src/hosted/manifest.ts b/src/hosted/manifest.ts index 50e4f5aa..17a24f84 100644 --- a/src/hosted/manifest.ts +++ b/src/hosted/manifest.ts @@ -11,6 +11,37 @@ import { type CommandListPresentation, } from '../command-presentation.js'; import type { HostedCommand, HostedManifest } from './types.js'; +import { webFetchCommand } from '../fetch/command.js'; + +const clientOwnedCommands: HostedCommand[] = [{ + site: webFetchCommand.site, + name: webFetchCommand.name, + ...(webFetchCommand.aliases ? { aliases: [...webFetchCommand.aliases] } : {}), + command: `${webFetchCommand.site}/${webFetchCommand.name}`, + description: webFetchCommand.description, + access: webFetchCommand.access, + strategy: (webFetchCommand.strategy ?? 'public').toUpperCase(), + browser: webFetchCommand.browser === true, + args: webFetchCommand.args.map(arg => ({ ...arg, ...(arg.choices ? { choices: [...arg.choices] } : {}) })), + columns: [...(webFetchCommand.columns ?? [])], + ...(webFetchCommand.tags ? { tags: [...webFetchCommand.tags] } : {}), + ...(webFetchCommand.keywords ? { keywords: [...webFetchCommand.keywords] } : {}), + ...(webFetchCommand.domain ? { domain: webFetchCommand.domain } : {}), + ...(webFetchCommand.defaultFormat ? { defaultFormat: webFetchCommand.defaultFormat } : {}), + ...(webFetchCommand.freshPage ? { freshPage: true } : {}), + clientOwned: true, +}]; + +export function withClientOwnedCommands(manifest: HostedManifest): HostedManifest { + const localCommandNames = new Set(clientOwnedCommands.map(command => command.command)); + return { + ...manifest, + commands: [ + ...manifest.commands.filter(command => !localCommandNames.has(command.command)), + ...clientOwnedCommands, + ], + }; +} export function isLocalOnlyHostedCommand(command: HostedCommand): boolean { return command.strategy.toUpperCase() === 'LOCAL'; @@ -33,11 +64,20 @@ export function presentHostedCommand(command: HostedCommand): PresentableCommand } export function hostedListRows(manifest: HostedManifest, structured: boolean): Record[] { - return commandListRows(hostedCommands(manifest).map(presentHostedCommand), structured); + return markClientOwned(commandListRows(hostedCommands(manifest).map(presentHostedCommand), structured), structured); } export function hostedListPresentation(manifest: HostedManifest, format: string): CommandListPresentation { - return commandListPresentation(hostedCommands(manifest).map(presentHostedCommand), format); + const presentation = commandListPresentation(hostedCommands(manifest).map(presentHostedCommand), format); + return { + ...presentation, + rows: markClientOwned(presentation.rows, presentation.structured), + }; +} + +function markClientOwned(rows: Record[], structured: boolean): Record[] { + if (!structured) return rows; + return rows.map(row => row.command === 'web/fetch' ? { ...row, clientOwned: true } : row); } export function siteNames(manifest: HostedManifest): string[] { diff --git a/src/hosted/root-command-surface.test.ts b/src/hosted/root-command-surface.test.ts index 6da56199..a3fe4686 100644 --- a/src/hosted/root-command-surface.test.ts +++ b/src/hosted/root-command-surface.test.ts @@ -193,6 +193,13 @@ describe('hosted root command surface', () => { }); }); + it('advertises client-owned web fetch at the hosted root', () => { + expect(HOSTED_ROOT_HELP.commands).toContainEqual({ + name: 'web', + description: 'Fetch URLs locally without launching a browser', + }); + }); + it('advertises hosted profile management as a root command, not a local-only namespace', () => { expect(HOSTED_ROOT_HELP.commands).toContainEqual({ name: 'profile', diff --git a/src/hosted/runner.test.ts b/src/hosted/runner.test.ts index 961c13b1..743dc638 100644 --- a/src/hosted/runner.test.ts +++ b/src/hosted/runner.test.ts @@ -261,6 +261,24 @@ function captureLocalBrowserStructure(argv: string[]): { } describe('runHostedCli', () => { + it('presents web fetch help from local metadata without dispatching it to Cloud', async () => { + const stdout = sink(); + const fetchImpl = vi.fn(async () => manifestResponse()); + + const result = await runHostedCli(['web', 'fetch', '--help'], { + config: makeHostedConfig({ apiBaseUrl: 'https://api.example.com', apiKey: 'key' }), + stdout: stdout.stream, + fetchImpl, + }); + + expect(result).toEqual({ handled: true, exitCode: 0 }); + expect(stdout.text()).toContain('--url '); + expect(stdout.text()).toContain('--timeout [value]'); + expect(stdout.text()).toContain('--max-chars [value]'); + expect(stdout.text()).toContain('--allow-private [value]'); + expect(fetchImpl.mock.calls.map(([url]) => String(url))).toEqual(['https://api.example.com/v1/manifest']); + }); + const publicProfile = { id: 'profile_work', name: 'Work', diff --git a/src/hosted/runner.ts b/src/hosted/runner.ts index f0a62753..00214d1c 100644 --- a/src/hosted/runner.ts +++ b/src/hosted/runner.ts @@ -45,6 +45,7 @@ import { isLocalOnlyHostedCommand, renderHostedCommandHelp, renderHostedSiteHelp, + withClientOwnedCommands, } from './manifest.js'; import { isHostedConfig, loadWebcmdConfig, type WebcmdConfig } from './config.js'; import { resolveHostedApiKey, type HostedCredentialStore } from './credentials.js'; @@ -163,8 +164,7 @@ async function dispatchHosted( return; } if (normalized.kind === 'completion') { - const manifest = await client.getManifest(); - validateManifestContractIdentity(manifest); + const manifest = await getPresentationManifest(client); await writeToStream(stdout, hostedCompletions(manifest, normalized.argv).join('\n') + '\n'); return; } @@ -217,8 +217,7 @@ async function dispatchHosted( await writeToStream(stdout, parsed.output); return; } - const manifest = await client.getManifest(); - validateManifestContractIdentity(manifest); + const manifest = await getPresentationManifest(client); await renderHostedList(manifest, parsed.format, parsed.formatExplicit, stdout, parsed.tag); return; } @@ -341,10 +340,9 @@ async function dispatchHosted( return; } - // The API manifest is tenant-scoped. Never merge package or local plugin - // commands into it: the installed package contract contains no site commands. - const manifest = await client.getManifest(); - validateManifestContractIdentity(manifest); + // The API manifest is tenant-scoped. Only the core client-owned presentation + // entry is merged; package and local plugin commands stay out. + const manifest = await getPresentationManifest(client); const site = args[0]!; const commandName = args[1]; @@ -405,6 +403,9 @@ async function dispatchHosted( await writeHostedHelp(stdout, args, hostedCommandHelpData(command), renderHostedCommandHelp(command)); return; } + if (command.clientOwned) { + throw new Error(`Internal invariant: client-owned command ${command.command} reached hosted dispatch.`); + } const startTime = now(); const response = hasPresentFileArgument(command, parsed.args) @@ -1195,7 +1196,7 @@ function hostedCompletions(manifest: HostedManifest, argv: string[]): string[] { hostedCommands(manifest), words, Number.isFinite(cursor) ? cursor! : words.length, - HOSTED_BUILTIN_COMMANDS, + HOSTED_BUILTIN_COMMANDS.filter(command => command !== 'web'), ); } @@ -1230,6 +1231,12 @@ function validateManifestContractIdentity(manifest: HostedManifest): void { } } +async function getPresentationManifest(client: HostedClient): Promise { + const manifest = await client.getManifest(); + validateManifestContractIdentity(manifest); + return withClientOwnedCommands(manifest); +} + function hostedContractCompatibilityLine(version: string): string | undefined { const match = /^(\d+)\.(\d+)\.(\d+)(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/.exec(version); if (!match) return undefined; diff --git a/src/hosted/types.ts b/src/hosted/types.ts index 6f61c02c..448bfa69 100644 --- a/src/hosted/types.ts +++ b/src/hosted/types.ts @@ -21,6 +21,7 @@ export interface HostedFileArgument { } export interface HostedCommand extends CommandSurfaceMetadata { + clientOwned?: boolean; site: string; name: string; aliases?: string[]; diff --git a/src/main.ts b/src/main.ts index b496b008..bf2aa703 100644 --- a/src/main.ts +++ b/src/main.ts @@ -23,6 +23,7 @@ import { PKG_VERSION } from './version.js'; import { EXIT_CODES } from './errors.js'; import { isSupportedNodeVersion, MIN_SUPPORTED_NODE_MAJOR } from './runtime-detect.js'; import { CONFIG_DIR_NAME } from './brand.js'; +import { parseHostedRootCommandSurface } from './root-command-surface.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -77,9 +78,9 @@ if (!fastPathHandled) { } else if (argv[0] === 'skills' || argv[0] === 'update') { const { createProgram } = await import('./cli.js'); await createProgram(BUILTIN_CLIS, USER_CLIS).parseAsync(argv, { from: 'user' }); - } else if (argv[0] === 'web' && argv[1] === 'fetch') { - const { runClientOwnedWebFetch } = await import('./fetch/command.js'); - await runClientOwnedWebFetch(argv); + } else if (isWebFetch(argv)) { + const { runWebFetchCommand } = await import('./fetch/command.js'); + await runWebFetchCommand(argv); } else { const { shouldUseHostedMode } = await import('./hosted/config.js'); if (shouldUseHostedMode()) { @@ -98,6 +99,15 @@ if (!fastPathHandled) { } } +function isWebFetch(args: readonly string[]): boolean { + try { + const parsed = parseHostedRootCommandSurface(args); + return parsed.kind === 'dispatch' && parsed.argv[0] === 'web' && parsed.argv[1] === 'fetch'; + } catch { + return false; + } +} + async function runLocalMain(): Promise { // Fast path: --get-completions — read from manifest, skip discovery const getCompIdx = process.argv.indexOf('--get-completions'); diff --git a/src/manifest-types.ts b/src/manifest-types.ts index 69a52d21..9d35adcf 100644 --- a/src/manifest-types.ts +++ b/src/manifest-types.ts @@ -40,6 +40,17 @@ export interface ManifestEntry { modulePath?: string; /** Relative path to the source file from clis/ dir (e.g. 'site/cmd.js') */ sourceFile?: string; + /** + * Package subpath export that registers this command, e.g. './fetch/command'. + * + * Present only on core-owned commands, which ship in the package's own + * `dist/` rather than as adapter files under `clis/`. Such an entry has no + * `modulePath`/`sourceFile`: there is no adapter file to resolve, and + * consumers that load adapters by path must import this export instead. + */ + packageExport?: string; + /** Command runs only in the local client, never in hosted execution. */ + clientOwned?: boolean; /** Pre-navigation control — see CliCommand.navigateBefore */ navigateBefore?: boolean | string; /** Site session lifecycle defaults — see CliCommand.siteSession */ diff --git a/src/output.ts b/src/output.ts index bd2f2776..471c1b6d 100644 --- a/src/output.ts +++ b/src/output.ts @@ -18,6 +18,8 @@ export interface RenderOptions { elapsed?: number; source?: string; footerExtra?: string; + /** Command-supplied markdown renderer — see CliCommand.renderMarkdown. */ + markdown?: (data: unknown) => string | undefined; } export interface ErrorRenderOptions { @@ -146,6 +148,8 @@ function formatPlain(data: unknown): string { } function formatMarkdown(data: unknown, opts: RenderOptions): string { + const custom = opts.markdown?.(data); + if (custom !== undefined) return custom.endsWith('\n') ? custom : `${custom}\n`; const rows = normalizeRows(data); if (!rows.length) return ''; if (rows.length === 1) { diff --git a/src/package-exports.test.ts b/src/package-exports.test.ts index 86ffaf89..d4afed18 100644 --- a/src/package-exports.test.ts +++ b/src/package-exports.test.ts @@ -1,69 +1,39 @@ /** * Regression tests for package exports. * - * Ensures adapter files use @agentrhq/webcmd public package imports - * (not fragile relative paths) and that all declared exports resolve - * to real files. + * Ensures no adapter tree sneaks back into the published package and that all + * declared exports resolve to real files. */ import { describe, it, expect } from 'vitest'; import * as fs from 'node:fs'; import * as path from 'node:path'; -import { builtinModules } from 'node:module'; import { fileURLToPath } from 'node:url'; -import ts from 'typescript'; +import { buildManifest, buildManifestArtifacts } from './build-manifest.js'; +import { getRegistry } from './registry.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(__dirname, '..'); -const CLIS_DIR = path.join(ROOT, 'clis', 'web'); +const CLIS_DIR = path.join(ROOT, 'clis'); const pkgJson = JSON.parse(fs.readFileSync(path.join(ROOT, 'package.json'), 'utf-8')); /** Recursively collect all JS adapter files in a directory. */ -function collectAdapterFiles(dir: string, opts?: { excludeTests?: boolean }): string[] { +function collectAdapterFiles(dir: string): string[] { const results: string[] = []; if (!fs.existsSync(dir)) return results; for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { const full = path.join(dir, entry.name); - if (entry.isDirectory()) { - results.push(...collectAdapterFiles(full, opts)); - } else if (entry.name.endsWith('.js') && !entry.name.endsWith('.d.js')) { - if (opts?.excludeTests && (entry.name.endsWith('.test.js') || entry.name.startsWith('test-'))) continue; - results.push(full); - } + if (entry.isDirectory()) results.push(...collectAdapterFiles(full)); + else if (entry.name.endsWith('.js') && !entry.name.endsWith('.d.js')) results.push(full); } return results; } -const ALLOWED_BARE_IMPORTS = new Set([ - '@agentrhq/webcmd', - ...builtinModules.flatMap((name) => name.startsWith('node:') - ? [name, name.slice(5)] - : [name, `node:${name}`]), -]); - -function isAllowedImport(specifier: string): boolean { - return specifier.startsWith('./') - || specifier.startsWith('../') - || specifier.startsWith('/') - || specifier.startsWith('@agentrhq/webcmd/') - || ALLOWED_BARE_IMPORTS.has(specifier); -} - -/** Forbidden relative import patterns that should have been replaced. - * Uses (?:\.\./)+ to catch any depth of ../ traversal. - * Covers: import/export from, vi.mock(), vi.importActual(). */ -const FORBIDDEN_PATTERNS = [ - /(?:from|mock|importActual)\s*\(?['"](?:\.\.\/)+src\//, - /(?:from|mock|importActual)\s*\(?['"](?:\.\.\/)+browser\//, - /(?:from|mock|importActual)\s*\(?['"](?:\.\.\/)+download\//, - /(?:from|mock|importActual)\s*\(?['"](?:\.\.\/)+pipeline\//, -]; - -describe('bundled web adapter imports use package exports', () => { - const adapterFiles = collectAdapterFiles(CLIS_DIR); - const runtimeAdapterFiles = collectAdapterFiles(CLIS_DIR, { excludeTests: true }); - - it('bundles the web adapter under clis/web', () => { - expect(adapterFiles.length).toBeGreaterThan(0); +describe('adapter packaging', () => { + // The `web` site used to live in clis/web. It is core TypeScript under + // src/fetch now, so import hygiene is enforced by tsc rather than by scanning + // adapter sources — but the packaging boundary below still needs asserting. + it('ships no bundled adapter tree', () => { + expect(collectAdapterFiles(CLIS_DIR)).toEqual([]); }); it('excludes adapters from package files and the install lifecycle', () => { @@ -73,41 +43,70 @@ describe('bundled web adapter imports use package exports', () => { expect(pkgJson.scripts.postinstall).not.toMatch(/fetch-adapters/); }); - it('no adapter uses relative imports to src/, browser/, download/, or pipeline/', () => { - const violations: string[] = []; - for (const file of adapterFiles) { - const content = fs.readFileSync(file, 'utf-8'); - for (const pattern of FORBIDDEN_PATTERNS) { - if (pattern.test(content)) { - const rel = path.relative(ROOT, file); - const match = content.match(pattern)?.[0]; - violations.push(`${rel}: ${match}`); - } - } + // packageExport supports package discovery and import verification for + // core-owned commands; it does not make them Cloud-executable. + it('every manifest entry is resolvable: a clis/ path or a real package export', () => { + const manifest = JSON.parse(fs.readFileSync(path.join(ROOT, 'cli-manifest.json'), 'utf-8')) as Array>; + const exports = pkgJson.exports as Record; + + expect(manifest.length).toBeGreaterThan(0); + for (const entry of manifest) { + const command = `${entry.site}/${entry.name}`; + if (entry.modulePath) continue; + expect(entry.packageExport, `${command} has neither modulePath nor packageExport`).toBeTruthy(); + expect(exports[entry.packageExport!], `${command} declares a packageExport missing from package.json exports`).toBeTruthy(); + const source = exports[entry.packageExport!]!.replace(/^\.\/dist\//, './').replace(/\.js$/, '.ts'); + expect(fs.existsSync(path.join(ROOT, source)), `${command} export has no source file`).toBe(true); } - expect(violations).toEqual([]); }); - it('non-test adapters only import node builtins, relative modules, or webcmd public APIs', () => { - const violations: Array<{ file: string; specifier: string }> = []; - - for (const file of runtimeAdapterFiles) { - const source = fs.readFileSync(file, 'utf-8'); - const module = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true, ts.ScriptKind.JS); - - for (const stmt of module.statements) { - if (!ts.isImportDeclaration(stmt) && !ts.isExportDeclaration(stmt)) continue; - const specifier = stmt.moduleSpecifier?.getText(module).slice(1, -1); - if (specifier && !isAllowedImport(specifier)) { - violations.push({ - file: path.relative(ROOT, file), - specifier, - }); - } - } - } + it('maps web/fetch to its source and built package export', () => { + const manifest = JSON.parse(fs.readFileSync(path.join(ROOT, 'cli-manifest.json'), 'utf-8')) as Array>; + const command = manifest.find(entry => entry.site === 'web' && entry.name === 'fetch'); + + expect(command).toMatchObject({ packageExport: './fetch/command' }); + expect(fs.existsSync(path.join(ROOT, 'src/fetch/command.ts'))).toBe(true); + expect((pkgJson.exports as Record)['./fetch/command']) + .toBe('./dist/src/fetch/command.js'); + }); + + it('generates only web/fetch and its command package export', async () => { + const { entries } = await buildManifest(); + const webCommands = entries + .filter(entry => entry.site === 'web') + .map(entry => `${entry.site}/${entry.name}`); + const registeredWebCommands = [...getRegistry().values()] + .filter(command => command.site === 'web') + .map(command => `${command.site}/${command.name}`) + .sort(); + const fetchExports = Object.keys(pkgJson.exports as Record) + .filter(exportPath => exportPath.startsWith('./fetch/')); + + expect(webCommands).toEqual(['web/fetch']); + expect(registeredWebCommands).toEqual(['web/fetch']); + expect(fetchExports).toEqual(['./fetch/command']); + }); - expect(violations).toEqual([]); + it('publishes only client-owned web/fetch in generated artifacts', async () => { + const { entries } = await buildManifest(); + const artifacts = buildManifestArtifacts(entries, String(pkgJson.version), []); + const manifest = JSON.parse(artifacts.manifestJson) as Array>; + const contract = JSON.parse(artifacts.hostedContractJson) as { + commands: Array>; + }; + const manifestEntries = manifest.filter(entry => entry.site === 'web'); + const contractEntries = contract.commands.filter(entry => entry.site === 'web'); + + expect(manifestEntries).toEqual([expect.objectContaining({ + name: 'fetch', clientOwned: true, packageExport: './fetch/command', + })]); + expect(manifestEntries[0]).not.toHaveProperty('modulePath'); + expect(manifestEntries[0]).not.toHaveProperty('sourceFile'); + expect(contractEntries).toEqual([expect.objectContaining({ + name: 'fetch', + sessionPolicy: 'local-only', + availability: { mode: 'local-only', reason: 'client-owned' }, + })]); }); }); diff --git a/src/registry.ts b/src/registry.ts index 07368968..c076391e 100644 --- a/src/registry.ts +++ b/src/registry.ts @@ -68,6 +68,15 @@ interface BaseCliCommand { /** Origin of this command: 'yaml', 'ts', or plugin name. */ source?: string; footerExtra?: (kwargs: CommandArgs) => string | undefined; + /** + * Render this command's result as a markdown document instead of the default + * key/value table. For commands whose payload *is* prose — a fetched page, an + * article — the table is unreadable and differs from what the command prints + * on any other path. + */ + renderMarkdown?: (data: unknown) => string | undefined; + /** Execute locally before either hosted or adapter-discovery mode boundary. */ + clientOwned?: boolean; validateArgs?: (kwargs: CommandArgs) => void; /** * Control pre-navigation and browser-session requirement. @@ -172,6 +181,8 @@ export function cli(opts: CliOptions): CliCommand { func: opts.func, pipeline: opts.pipeline, footerExtra: opts.footerExtra, + renderMarkdown: opts.renderMarkdown, + clientOwned: opts.clientOwned, validateArgs: opts.validateArgs, navigateBefore: opts.navigateBefore, siteSession: opts.siteSession, diff --git a/src/skills.test.ts b/src/skills.test.ts index 37499ce8..484ea0b5 100644 --- a/src/skills.test.ts +++ b/src/skills.test.ts @@ -50,19 +50,38 @@ function bundledSkill(name: string): string { } describe('webcmd skills content', () => { - it('keeps smart search on live discovery and explicit fetch escalation', () => { + it('keeps smart search on live discovery and explicit browser Sessions', () => { const skill = bundledSkill('smart-search'); + const browser = bundledSkill('webcmd-browser'); + const skills = [skill, browser]; + const sessionId = 'session_7d8f2c10-4a11-4f3e-9c22-1b6de0a91f45'; + const sessionWorkflow = [ + 'webcmd --profile work session create', + `webcmd --profile work --session ${sessionId} browser run --stdin`, + `webcmd --profile work --session ${sessionId} browser snapshot --snapshot-mode read`, + `webcmd --profile work session close ${sessionId}`, + ]; expect(skill).toContain('webcmd list --tag search -f json'); expect(skill).toContain('webcmd plugin search'); expect(skill).toContain('webcmd plugin install'); + expect(skill).toContain('webcmd web fetch --url'); expect(skill).toContain('FETCH_BLOCKED'); expect(skill).toContain('FETCH_REQUIRES_BROWSER'); - expect(skill).toContain('webcmd web fetch-browser'); + for (const guide of skills) { + const normalizedGuide = guide.replaceAll(/\\\n\s*/g, ' ').replaceAll(/ {2,}/g, ' '); + for (const command of sessionWorkflow) { + expect(normalizedGuide).toContain(command); + } + expect(guide).toMatch(/web fetch.*(?:remains|runs).*local/i); + expect(guide).toMatch(/web fetch.*never opens a browser/i); + expect(guide).toMatch(/local.*Cloak[\s\S]{0,160}hosted.*Webcmd Cloud.*Browser Use/i); + expect(guide).not.toMatch(/fetch-browser|web read|--browser/i); + } expect(skill).toContain('Search Summary'); expect(skill).toMatch(/at most three.*plugin/i); expect(skill).toMatch(/up to five.*candidate/i); expect(skill).toMatch(/three.*URL.*default/i); - expect(skill).toMatch(/two.*browser fetch/i); + expect(skill).toMatch(/two.*browser.*Session/i); expect(skill).toContain('## Site-named fast path'); expect(skill).toMatch(/cost order is mandatory when the request does not name a site/i); expect(skill).not.toContain('references/sources-'); @@ -117,9 +136,8 @@ describe('webcmd skills content', () => { const usage = bundledSkill('webcmd-usage'); const autofix = bundledSkill('webcmd-autofix'); const author = bundledSkill('webcmd-adapter-author'); - const browser = bundledSkill('webcmd-browser'); - const skills = [usage, autofix, author, browser]; - const handoffSkills = [usage, autofix, browser]; + const skills = [usage, autofix, author]; + const handoffSkills = [usage, autofix]; const autofixAuthRequired = autofix.match(/^- \*\*`AUTH_REQUIRED`\*\*[\s\S]*?(?=\n- \*\*)/m)?.[0] ?? ''; const autofixAuthRequiredRow = autofix.split('\n') .find((line) => line.startsWith('| AUTH_REQUIRED |')) ?? ''; diff --git a/tests/e2e/article-download-pipeline.test.ts b/tests/e2e/article-download-pipeline.test.ts deleted file mode 100644 index d59cd234..00000000 --- a/tests/e2e/article-download-pipeline.test.ts +++ /dev/null @@ -1,174 +0,0 @@ -/** - * E2E regression tests for the HTML → Markdown article pipeline. - * - * Drives real pages through `webcmd web read` and asserts the hardened - * converter's invariants hold on the produced file: - * - no base64 `data:image/…` leaks - * - no