From 35102c6e26b66280712b8df10379f390cfb98f19 Mon Sep 17 00:00:00 2001 From: Ankit Ranjan Date: Wed, 12 Aug 2026 17:36:21 +0530 Subject: [PATCH 01/24] refactor(fetch): move web into core and auto-escalate to the browser tier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `webcmd web` lived in two places: `web fetch` was a hardcoded fast path in main.ts, and `web fetch-browser` was an adapter in clis/web that the published tarball never shipped (`clis/` is not in package.json `files`). The split produced a recurring class of bug rather than isolated ones. The whole ladder now lives in src/fetch and ships in dist/: - `web fetch` walks plain HTTP -> impit -> browser in one command. A blocked page is rendered and returned instead of raising an error that names a second command. `--browser false` opts out; escalation is local-mode only, since hosted mode executes adapters server-side. - `web fetch-browser` keeps the article-export pipeline (--output, --download-images, --wait-for, --diagnose) for callers who want files. - Both are registered in the core registry, so help, `list`, completions and the manifests carry them with no plugin installed. - The fast path stays for plain fetches but hands `-h`/`-f`/`--trace` to the registered command, and renders the standard error envelope instead of leaking a raw Node stack trace. clis/web is deleted; build-manifest now emits core-registered commands with no modulePath, since there is no adapter file under clis/ to resolve. Also fixes the challenge classifier, which flattened every response header into one string and grepped it. A CSP naming cdnjs.cloudflare.com, or `server: cloudflare` on a healthy 200, was read as a bot challenge — so example.com and news.ycombinator.com both burned two retries and failed with FETCH_BLOCKED. Header evidence is now limited to headers describing the response itself, and markers are split into decisive (cf-mitigated, "just a moment") vs corroborating (cloudflare, captcha), the latter requiring a 403/429/503. Fixes #246, #247, #252, #264 Fixes #283 (classifier half; the safe-proxy EPIPE half landed in #265) Co-Authored-By: Claude Opus 5 --- cli-manifest.json | 57 ++- clis/web/README.md | 10 - clis/web/fetch.js | 1 - clis/web/test/fetch-browser.test.js | 392 --------------- docs/cli-reference.mdx | 16 +- scripts/silent-column-drop-baseline.json | 21 - scripts/typed-error-lint-baseline.json | 32 -- skills/smart-search/SKILL.md | 20 +- src/build-manifest.ts | 35 +- src/cli.ts | 4 + src/fetch/browser.test.ts | 440 +++++++++++++++++ .../fetch-browser.js => src/fetch/browser.ts | 448 ++++++++++-------- src/fetch/classify.test.ts | 54 +++ src/fetch/classify.ts | 35 +- src/fetch/client.ts | 10 +- src/fetch/command.test.ts | 105 +++- src/fetch/command.ts | 181 ++++++- src/fetch/extract.ts | 2 +- src/main.ts | 13 +- src/package-exports.test.ts | 93 +--- src/skills.test.ts | 11 +- 21 files changed, 1198 insertions(+), 782 deletions(-) delete mode 100644 clis/web/README.md delete mode 100644 clis/web/fetch.js delete mode 100644 clis/web/test/fetch-browser.test.js create mode 100644 src/fetch/browser.test.ts rename clis/web/fetch-browser.js => src/fetch/browser.ts (53%) diff --git a/cli-manifest.json b/cli-manifest.json index cc96cc2c..73c85b84 100644 --- a/cli-manifest.json +++ b/cli-manifest.json @@ -1,8 +1,61 @@ [ + { + "site": "web", + "name": "fetch", + "description": "Fetch a URL, escalating to a real browser only if plain HTTP is blocked", + "access": "read", + "strategy": "public", + "browser": false, + "args": [ + { + "name": "url", + "type": "string", + "required": true, + "help": "Any http or https URL" + }, + { + "name": "timeout", + "type": "int", + "default": 30, + "required": false, + "help": "Total budget in seconds across every tier" + }, + { + "name": "max-chars", + "type": "int", + "default": 50000, + "required": false, + "help": "Truncate content at this many characters (0 disables)" + }, + { + "name": "allow-private", + "type": "boolean", + "default": false, + "required": false, + "help": "Allow fetching private/loopback addresses" + }, + { + "name": "browser", + "type": "boolean", + "default": true, + "required": false, + "help": "Escalate to a real browser when the site blocks plain HTTP (--browser false to stop at HTTP)" + }, + { + "name": "wait", + "type": "int", + "default": 3, + "required": false, + "help": "Seconds to wait after page load when escalating to the browser" + } + ], + "defaultFormat": "md", + "type": "js" + }, { "site": "web", "name": "fetch-browser", - "description": "Fetch any web page and export as Markdown", + "description": "Fetch any web page in a real browser and export as Markdown", "access": "read", "strategy": "cookie", "browser": true, @@ -88,8 +141,6 @@ "saved" ], "type": "js", - "modulePath": "web/fetch-browser.js", - "sourceFile": "web/fetch-browser.js", "navigateBefore": false } ] diff --git a/clis/web/README.md b/clis/web/README.md deleted file mode 100644 index a4f69f9b..00000000 --- a/clis/web/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# webcmd web - -Bundled with `webcmd` by default. - -## Commands - -| Command | Description | -| --- | --- | -| `webcmd web fetch` | Fetch a URL locally without launching a browser | -| `webcmd web fetch-browser` | Fetch any web page and export as Markdown | diff --git a/clis/web/fetch.js b/clis/web/fetch.js deleted file mode 100644 index 113b6169..00000000 --- a/clis/web/fetch.js +++ /dev/null @@ -1 +0,0 @@ -import '@agentrhq/webcmd/fetch/command'; diff --git a/clis/web/test/fetch-browser.test.js b/clis/web/test/fetch-browser.test.js deleted file mode 100644 index 500a7ac3..00000000 --- a/clis/web/test/fetch-browser.test.js +++ /dev/null @@ -1,392 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { JSDOM } from 'jsdom'; - -const { mockDownloadArticle } = vi.hoisted(() => ({ - mockDownloadArticle: vi.fn(), -})); - -vi.mock('@agentrhq/webcmd/download/article-download', () => ({ - downloadArticle: mockDownloadArticle, -})); - -const { __test__ } = await import('../fetch-browser.js'); - -describe('web/fetch-browser stdout behavior', () => { - const read = __test__.command; - const extractedArticle = { - title: 'Example Article', - author: 'Author', - publishTime: '2026-04-22', - contentHtml: '

hello

', - imageUrls: ['https://example.com/a.jpg'], - diagnostics: { - url: 'https://example.com/article', - frames: [], - emptyContainers: [], - includedFrameCount: 0, - }, - }; - const page = { - goto: vi.fn().mockResolvedValue(undefined), - wait: vi.fn().mockResolvedValue(undefined), - evaluate: vi.fn().mockResolvedValue(extractedArticle), - startNetworkCapture: vi.fn().mockResolvedValue(true), - readNetworkCapture: vi.fn().mockResolvedValue([]), - }; - - beforeEach(() => { - vi.restoreAllMocks(); - vi.useRealTimers(); - mockDownloadArticle.mockReset(); - mockDownloadArticle.mockResolvedValue([{ - title: 'Example Article', - author: 'Author', - publish_time: '2026-04-22', - status: 'success', - size: '1 KB', - saved: '-', - }]); - page.goto.mockClear(); - page.wait.mockClear(); - page.evaluate.mockClear(); - page.evaluate.mockResolvedValue(extractedArticle); - page.startNetworkCapture.mockClear(); - page.startNetworkCapture.mockResolvedValue(true); - page.readNetworkCapture.mockClear(); - page.readNetworkCapture.mockResolvedValue([]); - }); - - it('returns null in --stdout mode so the CLI does not append result rows to stdout', async () => { - const result = await read.func(page, { - url: 'https://example.com/article', - output: '/tmp/out', - 'download-images': false, - stdout: true, - }); - - expect(result).toBeNull(); - expect(mockDownloadArticle).toHaveBeenCalledWith( - expect.objectContaining({ - title: 'Example Article', - sourceUrl: 'https://example.com/article', - }), - expect.objectContaining({ - output: '/tmp/out', - stdout: true, - }), - ); - expect(page.evaluate.mock.calls[0]?.[0]).toContain('const frameMode = "same-origin"'); - }); - - it('still returns the saved-row payload when writing to disk', async () => { - const rows = [{ title: 'Example Article', saved: '/tmp/out/Example Article/example.md' }]; - mockDownloadArticle.mockResolvedValue(rows); - - const result = await read.func(page, { - url: 'https://example.com/article', - output: '/tmp/out', - 'download-images': false, - stdout: false, - }); - - expect(result).toBe(rows); - }); - - it('waits for a selector in the main document or same-origin iframes before extracting', async () => { - page.evaluate - .mockResolvedValueOnce({ ok: true, scope: 'iframe', url: 'https://example.com/frame' }) - .mockResolvedValueOnce(extractedArticle); - - await read.func(page, { - url: 'https://example.com/article', - output: '/tmp/out', - 'download-images': false, - 'wait-for': '#gridDatas li', - wait: 7, - stdout: false, - }); - - expect(page.wait).not.toHaveBeenCalled(); - expect(page.evaluate).toHaveBeenCalledTimes(2); - expect(page.evaluate.mock.calls[0]?.[0]).toContain('"#gridDatas li"'); - expect(page.evaluate.mock.calls[0]?.[0]).toContain('sameOriginFrameDocs'); - expect(page.evaluate.mock.calls[1]?.[0]).toContain('const frameMode = "same-origin"'); - }); - - it('throws a clear error when --wait-for times out', async () => { - page.evaluate.mockResolvedValueOnce({ ok: false, timedOut: true, selector: '#missing' }); - - await expect(read.func(page, { - url: 'https://example.com/article', - output: '/tmp/out', - 'download-images': false, - 'wait-for': '#missing', - wait: 1, - stdout: false, - })).rejects.toThrow('Timed out waiting for selector "#missing"'); - - expect(mockDownloadArticle).not.toHaveBeenCalled(); - }); - - it('starts network capture and writes diagnostics in diagnose mode', async () => { - const stderr = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); - page.readNetworkCapture.mockResolvedValueOnce([{ - method: 'POST', - url: 'https://example.com/api/data', - responseStatus: 200, - responseContentType: 'application/json', - responsePreview: '{"ok":true}', - }]); - - await read.func(page, { - url: 'https://example.com/article', - output: '/tmp/out', - 'download-images': false, - diagnose: true, - wait: 0, - stdout: false, - }); - - expect(page.startNetworkCapture).toHaveBeenCalledWith(''); - expect(page.readNetworkCapture).toHaveBeenCalledTimes(1); - expect(stderr).toHaveBeenCalledWith(expect.stringContaining('[web-fetch-browser diagnose]')); - expect(stderr).toHaveBeenCalledWith(expect.stringContaining('POST 200 application/json https://example.com/api/data')); - }); - - it('passes --frames none into the extractor', async () => { - await read.func(page, { - url: 'https://example.com/article', - output: '/tmp/out', - 'download-images': false, - frames: 'none', - stdout: false, - }); - - expect(page.evaluate.mock.calls[0]?.[0]).toContain('const frameMode = "none"'); - }); - - it('passes --frames all-same-origin into the extractor', async () => { - await read.func(page, { - url: 'https://example.com/article', - output: '/tmp/out', - 'download-images': false, - frames: 'all-same-origin', - stdout: false, - }); - - expect(page.evaluate.mock.calls[0]?.[0]).toContain('const frameMode = "all-same-origin"'); - }); - - it('fails fast when --wait-until networkidle is requested but capture is unavailable', async () => { - page.startNetworkCapture.mockResolvedValue(false); - - await expect(read.func(page, { - url: 'https://example.com/article', - output: '/tmp/out', - 'download-images': false, - 'wait-until': 'networkidle', - wait: 2, - stdout: false, - })).rejects.toThrow('Network capture is unavailable'); - - expect(page.wait).not.toHaveBeenCalled(); - expect(mockDownloadArticle).not.toHaveBeenCalled(); - }); - - it('fails fast when network traffic never settles before the networkidle timeout', async () => { - vi.useFakeTimers(); - page.readNetworkCapture.mockResolvedValue([{ - method: 'POST', - url: 'https://example.com/api/data', - responseStatus: 200, - responseContentType: 'application/json', - responsePreview: '{"ok":true}', - }]); - - const pending = expect(read.func(page, { - url: 'https://example.com/article', - output: '/tmp/out', - 'download-images': false, - 'wait-until': 'networkidle', - wait: 1, - stdout: false, - })).rejects.toThrow('Timed out waiting for network idle after 1s'); - - await vi.advanceTimersByTimeAsync(2000); - - await pending; - expect(mockDownloadArticle).not.toHaveBeenCalled(); - }); -}); - -describe('web/fetch-browser render-aware helpers', () => { - it('merges accessible same-origin iframe bodies into the extracted HTML', () => { - const dom = new JSDOM(` -
-

Shell

- -
- `, { url: 'https://example.com/main.html', runScripts: 'outside-only' }); - const frame = dom.window.document.querySelector('iframe'); - frame.contentDocument.open(); - frame.contentDocument.write('
Name

Station A 42

'); - frame.contentDocument.close(); - - const result = dom.window.eval(__test__.buildRenderAwareExtractorJs({ frames: 'same-origin' })); - - expect(result.diagnostics.includedFrameCount).toBe(1); - expect(result.contentHtml).toContain('data-webcmd-iframe-source="https://example.com/frame.html"'); - expect(result.contentHtml).toContain('localized text iframe: https://example.com/frame.html'); - expect(result.contentHtml).toContain('Station A 42'); - expect(result.diagnostics.emptyContainers).toEqual(expect.arrayContaining([ - expect.objectContaining({ scope: 'iframe', id: 'gridDatas', url: 'https://example.com/frame.html' }), - ])); - expect(result.diagnostics.emptyContainers.every(item => item.scope === 'iframe')).toBe(true); - }); - - it('merges readable same-origin iframes outside the selected content element', () => { - const dom = new JSDOM(` -
-

Main Article

-

${'Main content '.repeat(30)}

-
- - `, { url: 'https://example.com/main.html', runScripts: 'outside-only' }); - const frame = dom.window.document.querySelector('iframe'); - frame.contentDocument.open(); - frame.contentDocument.write(`

Outside Frame

${'Frame data '.repeat(12)}

`); - frame.contentDocument.close(); - - const result = dom.window.eval(__test__.buildRenderAwareExtractorJs({ frames: 'same-origin' })); - - expect(result.diagnostics.includedFrameCount).toBe(1); - expect(result.contentHtml).toContain('data-webcmd-iframe-source="https://example.com/outside.html"'); - expect(result.contentHtml).toContain('Outside Frame'); - expect(result.contentHtml).toContain('Frame data'); - }); - - it('keeps short data-like iframes outside the selected content element', () => { - const dom = new JSDOM(` -
-

Main Article

-

${'Main content '.repeat(30)}

-
- - `, { url: 'https://example.com/main.html', runScripts: 'outside-only' }); - const frame = dom.window.document.querySelector('iframe'); - frame.contentDocument.open(); - frame.contentDocument.write('
water level
42
'); - frame.contentDocument.close(); - - const result = dom.window.eval(__test__.buildRenderAwareExtractorJs({ frames: 'same-origin' })); - - expect(result.diagnostics.includedFrameCount).toBe(1); - expect(result.contentHtml).toContain('42'); - expect(result.diagnostics.emptyContainers).toEqual(expect.arrayContaining([ - expect.objectContaining({ scope: 'iframe', id: 'gridDatas', url: 'https://example.com/data.html' }), - ])); - expect(result.diagnostics.emptyContainers.every(item => item.scope === 'iframe')).toBe(true); - }); - - it('skips short non-structural iframes outside the selected content element', () => { - const dom = new JSDOM(` -
-

Main Article

-

${'Main content '.repeat(30)}

-
- - `, { url: 'https://example.com/main.html', runScripts: 'outside-only' }); - const frame = dom.window.document.querySelector('iframe'); - frame.contentDocument.open(); - frame.contentDocument.write('

tiny note

'); - frame.contentDocument.close(); - - const result = dom.window.eval(__test__.buildRenderAwareExtractorJs({ frames: 'same-origin' })); - - expect(result.diagnostics.includedFrameCount).toBe(0); - expect(result.contentHtml).not.toContain('tiny note'); - }); - - it('includes short non-structural iframes in all-same-origin mode', () => { - const dom = new JSDOM(` -
-

Main Article

-

${'Main content '.repeat(30)}

-
- - `, { url: 'https://example.com/main.html', runScripts: 'outside-only' }); - const frame = dom.window.document.querySelector('iframe'); - frame.contentDocument.open(); - frame.contentDocument.write('
Online: 42°C
'); - frame.contentDocument.close(); - - const result = dom.window.eval(__test__.buildRenderAwareExtractorJs({ frames: 'all-same-origin' })); - - expect(result.diagnostics.includedFrameCount).toBe(1); - expect(result.contentHtml).toContain('Online: 42°C'); - }); - - it('marks API-like network entries as interesting and ignores static assets', () => { - expect(__test__.isInterestingNetworkEntry({ - method: 'POST', - url: 'https://example.com/GJZ/Ajax/Publish.ashx', - status: 200, - contentType: 'text/html', - size: 100, - bodyTruncated: false, - })).toBe(true); - expect(__test__.isInterestingNetworkEntry({ - method: 'POST', - url: 'https://example.com/GJZ/Ajax/Publish.ashx', - status: 200, - contentType: 'application/json', - size: 100, - bodyTruncated: false, - })).toBe(true); - expect(__test__.isInterestingNetworkEntry({ - method: 'GET', - url: 'https://example.com/app.js', - status: 200, - contentType: 'application/javascript', - size: 100, - bodyTruncated: false, - })).toBe(false); - }); - - it('formats frame and XHR diagnostics for shell pages', () => { - const output = __test__.formatDiagnostics({ - diagnostics: { - url: 'https://example.com/main.html', - includedFrameCount: 1, - frames: [{ - index: 0, - src: 'https://example.com/frame.html', - sameOrigin: true, - accessible: true, - textLength: 42, - }], - emptyContainers: [{ - scope: 'iframe', - url: 'https://example.com/frame.html', - tag: 'ul', - id: 'gridDatas', - className: '', - }], - }, - }, [{ - method: 'POST', - url: 'https://example.com/GJZ/Ajax/Publish.ashx', - status: 200, - contentType: 'application/json', - size: 64, - bodyTruncated: false, - }], true); - - expect(output).toContain('frames: 1, included_same_origin: 1'); - expect(output).toContain('[frame 0] same-origin accessible text=42 https://example.com/frame.html'); - expect(output).toContain('iframe: ul#gridDatas (https://example.com/frame.html)'); - expect(output).toContain('POST 200 application/json https://example.com/GJZ/Ajax/Publish.ashx'); - }); -}); diff --git a/docs/cli-reference.mdx b/docs/cli-reference.mdx index 3e5bee3d..7b26b89c 100644 --- a/docs/cli-reference.mdx +++ b/docs/cli-reference.mdx @@ -39,7 +39,7 @@ Agents should prefer existing adapters before raw browser exploration. For searc ## Search and Fetch -Use `webcmd web fetch` for direct URL fetches and fetch-first web search. It runs locally and does not launch a browser, including when Webcmd is configured for hosted mode. +Use `webcmd web fetch` for direct URL fetches and fetch-first web search. It is built into the CLI, so it works on a fresh install with no plugins. ```bash webcmd web fetch --url https://example.com/article @@ -47,10 +47,20 @@ webcmd web fetch --url "https://duckduckgo.com/html/?q=agentic%20browser%20autom webcmd web fetch --url "https://www.bing.com/search?q=agentic%20browser%20automation" ``` -Use `webcmd web fetch-browser` only when `web fetch` reports that browser rendering is required or the page is blocked: +`web fetch` tries plain HTTP first, then a browser-impersonating TLS client. If the site blocks both, it escalates to a real browser on its own and returns the rendered page — you do not have to notice the failure and retry. The result reports which tier answered: + +``` +Extraction: readability # plain HTTP or impit +Extraction: browser # escalated +``` + +Pass `--browser false` to stop at HTTP and get the `FETCH_BLOCKED` / `FETCH_REQUIRES_BROWSER` error instead. Escalation is local-mode only; in hosted mode the error is returned unchanged. + +Use `webcmd web fetch-browser` directly when you want the browser tier's article export — it writes Markdown plus downloaded images to a directory, and takes render controls `web fetch` does not expose: ```bash -webcmd web fetch-browser --url https://example.com/app-shell +webcmd web fetch-browser --url https://example.com/app-shell --output ./articles +webcmd web fetch-browser --url https://example.com/grid --wait-for "#results tr" --diagnose true ``` The old `web read` command has been renamed to `web fetch-browser`. diff --git a/scripts/silent-column-drop-baseline.json b/scripts/silent-column-drop-baseline.json index d9ef6cf4..aaf3b402 100644 --- a/scripts/silent-column-drop-baseline.json +++ b/scripts/silent-column-drop-baseline.json @@ -394,27 +394,6 @@ "y" ] }, - { - "command": "web/fetch-browser", - "file": "clis/web/fetch-browser.js", - "missing": [ - "accessible", - "index", - "sameOrigin", - "src", - "textLength" - ] - }, - { - "command": "web/fetch-browser", - "file": "clis/web/fetch-browser.js", - "missing": [ - "bodyTruncated", - "contentType", - "method", - "url" - ] - }, { "command": "yahoo-finance/quote", "file": "plugins/yahoo-finance/quote.js", diff --git a/scripts/typed-error-lint-baseline.json b/scripts/typed-error-lint-baseline.json index df711138..facb7a41 100644 --- a/scripts/typed-error-lint-baseline.json +++ b/scripts/typed-error-lint-baseline.json @@ -527,38 +527,6 @@ "text": "const screenName = user?.legacy?.screen_name || user?.core?.screen_name || 'unknown';", "occurrence": 0 }, - { - "rule": "silent-sentinel", - "command": "web/fetch-browser", - "file": "clis/web/fetch-browser.js", - "line": 378, - "text": "lines.push(` [frame ${frame.index}] ${frame.sameOrigin ? 'same-origin' : 'cross-origin'} ${frame.accessible ? 'accessible' : 'blocked'} text=${frame.textLength || 0} ${frame.src || '-'}`);", - "occurrence": 0 - }, - { - "rule": "silent-sentinel", - "command": "web/fetch-browser", - "file": "clis/web/fetch-browser.js", - "line": 390, - "text": "lines.push(` ${entry.method} ${entry.status || '-'} ${entry.contentType || '-'} ${entry.url}`);", - "occurrence": 0 - }, - { - "rule": "silent-sentinel", - "command": "web/fetch-browser", - "file": "clis/web/fetch-browser.js", - "line": 384, - "text": "lines.push(` ${item.scope}: ${selector} (${item.url || '-'})`);", - "occurrence": 0 - }, - { - "rule": "silent-sentinel", - "command": "web/fetch-browser", - "file": "clis/web/fetch-browser.js", - "line": 375, - "text": "lines.push(`url: ${diag.url || '-'}`);", - "occurrence": 0 - }, { "rule": "silent-sentinel", "command": "yollomi/edit", diff --git a/skills/smart-search/SKILL.md b/skills/smart-search/SKILL.md index 2955c175..7a679d7f 100644 --- a/skills/smart-search/SKILL.md +++ b/skills/smart-search/SKILL.md @@ -11,7 +11,7 @@ Use live fetch results, command metadata, and command help. Do not infer command Do not use this skill for plugin inventory, plugin management, or listing available extensions. Marketplace commands appear here only to find and install search-capable adapters needed for the current search/fetch task. -Cost order is mandatory when the request does not name a site: `webcmd web fetch` first, `webcmd web fetch-browser` second, search adapters last. Do not call search adapters until plain HTTP/TLS fetch and browser fetch cannot satisfy the task. +Cost order is mandatory when the request does not name a site: `webcmd web fetch` first, search adapters last. `web fetch` walks the HTTP → TLS → browser ladder itself, so one call already covers plain and browser fetching. Do not call search adapters until it has failed. When the request does name a site or community, take the site-native fast path below instead. @@ -35,21 +35,15 @@ Prefer primary sources, official docs, and direct content over search snippets. ## Direct URL -For a supplied HTTP(S) URL, fetch it first: +For a supplied HTTP(S) URL, fetch it: ```bash webcmd web fetch --url ``` -Only when the structured error code is `FETCH_BLOCKED` or `FETCH_REQUIRES_BROWSER`, use: +That single call handles browser escalation itself: if the site blocks plain HTTP and TLS impersonation, it renders the page in a browser and returns the content, reporting `Extraction: browser`. Do not chase a `FETCH_BLOCKED` with a second command — if you received that error, the browser tier already ran or was unavailable. -```bash -webcmd web fetch-browser --url -``` - -Do not escalate on message prose and do not make `web fetch` launch a browser. - -If direct fetch is rate-limited, blocked, CAPTCHA-gated, login-gated, geo-gated, or returns unusable extracted text, report that state. Only browser-escalate for `FETCH_BLOCKED` or `FETCH_REQUIRES_BROWSER`. +If the fetch is rate-limited, login-gated, geo-gated, or returns unusable extracted text, report that state rather than retrying the same URL. ## Fetch-first web search @@ -67,7 +61,7 @@ Query terms that collide with everyday English (`puppeteer`, `playwright`, `rust Extract useful result URLs from the fetched page and then fetch the target pages with `webcmd web fetch`. Search snippets and result titles are discovery only, not evidence. A page that yields zero usable result URLs is a failed search, not a search with no results: move to the next engine. -If the search-engine result page itself needs browser rendering, use at most one browser fetch for a search results page before trying another search engine. A recognised block, CAPTCHA, or challenge page retires that engine for this request: do not re-fetch variants of the same engine. Do not jump to adapters because one engine blocked, unless the request names a site. +If the search-engine result page itself needs browser rendering, `web fetch` escalates once on its own; do not re-run it for the same engine. A recognised block, CAPTCHA, or challenge page retires that engine for this request: do not re-fetch variants of the same engine. Do not jump to adapters because one engine blocked, unless the request names a site. ## Fetch evidence @@ -77,7 +71,7 @@ Fetch up to three result URLs by default (five for a broad comparison): webcmd web fetch --url ``` -Use up to two browser fetches by default, only when target-page `web fetch` returns `FETCH_BLOCKED` or `FETCH_REQUIRES_BROWSER`. Cite or link the source URL with substantive claims. +Browser escalation happens inside `web fetch`, so a blocked page costs one command, not two. Pass `--browser false` when you want a cheap HTTP-only probe and are willing to skip blocked pages. Cite or link the source URL with substantive claims. If fetch is rate-limited, auth-gated, CAPTCHA-gated, bot-detected, quota-limited, or geo-blocked, do not loop. Try another relevant URL/source when available; otherwise report the blocker. @@ -120,7 +114,7 @@ Do not add custom marketplaces in this workflow. In hosted mode, only verified h - One fetched search-engine page by default; second if weak/blocked; third only if the first two fail. - Up to five candidate commands before choosing. - Three URLs by default; five only for broad comparison. -- Two browser fetches by default. +- Two escalated (browser-tier) fetches by default; use `--browser false` once that budget is spent. - One adapter search by default; second only for weakness or corroboration. - Do not retry the same blocked command more than once. diff --git a/src/build-manifest.ts b/src/build-manifest.ts index 5cb93350..1ddafaa7 100644 --- a/src/build-manifest.ts +++ b/src/build-manifest.ts @@ -111,7 +111,7 @@ function isCliCommandValue(value: unknown, site: string): value is CliCommand { && Array.isArray(value.args); } -function toManifestEntry(cmd: CliCommand, modulePath: string, sourceFile?: string): ManifestEntry { +function toManifestEntry(cmd: CliCommand, modulePath?: string, sourceFile?: string): ManifestEntry { return { site: cmd.site, name: cmd.name, @@ -247,8 +247,39 @@ export async function scanClisDir( return { entries, failures }; } +/** + * Sites the core package owns and ships in `dist/`, rather than as adapter + * files under `clis/`. `web` moved here so the whole fetch ladder installs by + * default instead of living in a plugin nobody had (#247). + */ +const CORE_SITES = new Set(['web']); + +/** + * Manifest entries for core-registered commands. + * + * These deliberately carry no `modulePath`/`sourceFile`: there is no adapter + * file under `clis/` to resolve, and claiming one would point every consumer at + * a path that does not exist in the published tarball. Consumers that load + * adapters by path must treat a missing `modulePath` as "core-owned, import it + * from the package itself". + */ +export async function coreCommandEntries( + importer: (moduleHref: string) => Promise = moduleHref => import(moduleHref), +): Promise { + await importer(pathToFileURL(path.join(PACKAGE_ROOT, 'src/fetch/command.ts')).href); + return [...getRegistry().values()] + .filter(cmd => CORE_SITES.has(cmd.site)) + .sort((a, b) => a.site.localeCompare(b.site) || a.name.localeCompare(b.name)) + .map(cmd => toManifestEntry(cmd)); +} + export async function buildManifest(): Promise { - return scanClisDir(LEGACY_CLIS_DIR); + const scanned = await scanClisDir(LEGACY_CLIS_DIR); + const core = await coreCommandEntries(); + const entries = [...scanned.entries, ...core].sort( + (a, b) => a.site.localeCompare(b.site) || a.name.localeCompare(b.name), + ); + return { entries, failures: scanned.failures }; } export function serializeManifest(manifest: ManifestEntry[]): string { diff --git a/src/cli.ts b/src/cli.ts index 78cfc87f..13fad845 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -13,6 +13,10 @@ import { fileURLToPath, pathToFileURL } from 'node:url'; import { Command, Option } from 'commander'; import { findPackageRoot, getBuiltEntryCandidates } from './package-paths.js'; import { type CliCommand, getRegistry } from './registry.js'; +// Side-effect import: registers `web fetch` / `web fetch-browser` in the core +// registry so they reach help, `list`, completions and the manifests without a +// plugin install (#252, #247). Both tiers load their implementations lazily. +import './fetch/command.js'; import { commandListPresentation, filterCommandsByTag, toPresentableCommand } from './command-presentation.js'; import { configureCompletionCommandSurface, configureListCommandSurface, configurePluginInstallSurface, configurePluginSearchSurface } from './builtin-command-surface.js'; import { render as renderOutput } from './output.js'; diff --git a/src/fetch/browser.test.ts b/src/fetch/browser.test.ts new file mode 100644 index 00000000..1c018a95 --- /dev/null +++ b/src/fetch/browser.test.ts @@ -0,0 +1,440 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { JSDOM } from 'jsdom'; +import type { IPage } from '../types.js'; +import type { ExtractedPage } from './browser.js'; + +const { mockDownloadArticle } = vi.hoisted(() => ({ + mockDownloadArticle: vi.fn(), +})); + +vi.mock('../download/article-download.js', () => ({ + downloadArticle: mockDownloadArticle, +})); + +const { + buildRenderAwareExtractorJs, + formatDiagnostics, + isInterestingNetworkEntry, + normalizeFrameMode, + normalizeWaitUntil, + runFetchBrowser, +} = await import('./browser.js'); + +describe('web fetch browser tier stdout behavior', () => { + const extractedArticle = { + title: 'Example Article', + author: 'Author', + publishTime: '2026-04-22', + contentHtml: '

hello

', + imageUrls: ['https://example.com/a.jpg'], + diagnostics: { + url: 'https://example.com/article', + frames: [], + emptyContainers: [], + includedFrameCount: 0, + }, + }; + const page = { + goto: vi.fn().mockResolvedValue(undefined), + wait: vi.fn().mockResolvedValue(undefined), + evaluate: vi.fn().mockResolvedValue(extractedArticle), + startNetworkCapture: vi.fn().mockResolvedValue(true), + readNetworkCapture: vi.fn().mockResolvedValue([]), + }; + const run = (kwargs: Record) => runFetchBrowser(page as unknown as IPage, kwargs); + + beforeEach(() => { + vi.restoreAllMocks(); + vi.useRealTimers(); + mockDownloadArticle.mockReset(); + mockDownloadArticle.mockResolvedValue([{ + title: 'Example Article', + author: 'Author', + publish_time: '2026-04-22', + status: 'success', + size: '1 KB', + saved: '-', + }]); + page.goto.mockClear(); + page.wait.mockClear(); + page.evaluate.mockClear(); + page.evaluate.mockResolvedValue(extractedArticle); + page.startNetworkCapture.mockClear(); + page.startNetworkCapture.mockResolvedValue(true); + page.readNetworkCapture.mockClear(); + page.readNetworkCapture.mockResolvedValue([]); + }); + + it('returns null in --stdout mode so the CLI does not append result rows to stdout', async () => { + const result = await run({ + url: 'https://example.com/article', + output: '/tmp/out', + 'download-images': false, + stdout: true, + }); + + expect(result).toBeNull(); + expect(mockDownloadArticle).toHaveBeenCalledWith( + expect.objectContaining({ + title: 'Example Article', + sourceUrl: 'https://example.com/article', + }), + expect.objectContaining({ + output: '/tmp/out', + stdout: true, + }), + ); + expect(page.evaluate.mock.calls[0]?.[0]).toContain('const frameMode = "same-origin"'); + }); + + it('still returns the saved-row payload when writing to disk', async () => { + const rows = [{ title: 'Example Article', saved: '/tmp/out/Example Article/example.md' }]; + mockDownloadArticle.mockResolvedValue(rows); + + const result = await run({ + url: 'https://example.com/article', + output: '/tmp/out', + 'download-images': false, + stdout: false, + }); + + expect(result).toBe(rows); + }); + + it('waits for a selector in the main document or same-origin iframes before extracting', async () => { + page.evaluate + .mockResolvedValueOnce({ ok: true, scope: 'iframe', url: 'https://example.com/frame' }) + .mockResolvedValueOnce(extractedArticle); + + await run({ + url: 'https://example.com/article', + output: '/tmp/out', + 'download-images': false, + 'wait-for': '#gridDatas li', + wait: 7, + stdout: false, + }); + + expect(page.wait).not.toHaveBeenCalled(); + expect(page.evaluate).toHaveBeenCalledTimes(2); + expect(page.evaluate.mock.calls[0]?.[0]).toContain('"#gridDatas li"'); + expect(page.evaluate.mock.calls[0]?.[0]).toContain('sameOriginFrameDocs'); + expect(page.evaluate.mock.calls[1]?.[0]).toContain('const frameMode = "same-origin"'); + }); + + it('throws a clear error when --wait-for times out', async () => { + page.evaluate.mockResolvedValueOnce({ ok: false, timedOut: true, selector: '#missing' }); + + await expect(run({ + url: 'https://example.com/article', + output: '/tmp/out', + 'download-images': false, + 'wait-for': '#missing', + wait: 1, + stdout: false, + })).rejects.toThrow('Timed out waiting for selector "#missing"'); + + expect(mockDownloadArticle).not.toHaveBeenCalled(); + }); + + it('starts network capture and writes diagnostics in diagnose mode', async () => { + const stderr = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + page.readNetworkCapture.mockResolvedValueOnce([{ + method: 'POST', + url: 'https://example.com/api/data', + responseStatus: 200, + responseContentType: 'application/json', + responsePreview: '{"ok":true}', + }]); + + await run({ + url: 'https://example.com/article', + output: '/tmp/out', + 'download-images': false, + diagnose: true, + wait: 0, + stdout: false, + }); + + expect(page.startNetworkCapture).toHaveBeenCalledWith(''); + expect(page.readNetworkCapture).toHaveBeenCalledTimes(1); + expect(stderr).toHaveBeenCalledWith(expect.stringContaining('[web-fetch-browser diagnose]')); + expect(stderr).toHaveBeenCalledWith(expect.stringContaining('POST 200 application/json https://example.com/api/data')); + }); + + // A challenge interstitial hands off to the real page mid-evaluate. That is + // the normal case on the escalation path, so it must not surface as an error. + it('retries the extractor once when the page navigates out from under it', async () => { + page.evaluate + .mockRejectedValueOnce(new Error('page.evaluate: Execution context was destroyed, most likely because of a navigation')) + .mockResolvedValueOnce(extractedArticle); + + const result = await run({ url: 'https://challenged.example', output: '/tmp/out', 'download-images': false, stdout: false }); + + expect(page.evaluate).toHaveBeenCalledTimes(2); + expect(mockDownloadArticle).toHaveBeenCalledOnce(); + expect(result).not.toBeNull(); + }); + + it('rethrows a second navigation failure instead of looping', async () => { + const destroyed = new Error('Execution context was destroyed'); + page.evaluate.mockRejectedValueOnce(destroyed).mockRejectedValueOnce(destroyed); + + await expect(run({ url: 'https://challenged.example', output: '/tmp/out', 'download-images': false, stdout: false })) + .rejects.toThrow('Execution context was destroyed'); + expect(page.evaluate).toHaveBeenCalledTimes(2); + }); + + it('does not retry an unrelated evaluate failure', async () => { + page.evaluate.mockRejectedValueOnce(new Error('SyntaxError: bad selector')); + + await expect(run({ url: 'https://example.com/article', output: '/tmp/out', 'download-images': false, stdout: false })) + .rejects.toThrow('SyntaxError'); + expect(page.evaluate).toHaveBeenCalledTimes(1); + }); + + it('passes --frames none into the extractor', async () => { + await run({ + url: 'https://example.com/article', + output: '/tmp/out', + 'download-images': false, + frames: 'none', + stdout: false, + }); + + expect(page.evaluate.mock.calls[0]?.[0]).toContain('const frameMode = "none"'); + }); + + it('passes --frames all-same-origin into the extractor', async () => { + await run({ + url: 'https://example.com/article', + output: '/tmp/out', + 'download-images': false, + frames: 'all-same-origin', + stdout: false, + }); + + expect(page.evaluate.mock.calls[0]?.[0]).toContain('const frameMode = "all-same-origin"'); + }); + + it('fails fast when --wait-until networkidle is requested but capture is unavailable', async () => { + page.startNetworkCapture.mockResolvedValue(false); + + await expect(run({ + url: 'https://example.com/article', + output: '/tmp/out', + 'download-images': false, + 'wait-until': 'networkidle', + wait: 2, + stdout: false, + })).rejects.toThrow('Network capture is unavailable'); + + expect(page.wait).not.toHaveBeenCalled(); + expect(mockDownloadArticle).not.toHaveBeenCalled(); + }); + + it('fails fast when network traffic never settles before the networkidle timeout', async () => { + vi.useFakeTimers(); + page.readNetworkCapture.mockResolvedValue([{ + method: 'POST', + url: 'https://example.com/api/data', + responseStatus: 200, + responseContentType: 'application/json', + responsePreview: '{"ok":true}', + }]); + + const pending = expect(run({ + url: 'https://example.com/article', + output: '/tmp/out', + 'download-images': false, + 'wait-until': 'networkidle', + wait: 1, + stdout: false, + })).rejects.toThrow('Timed out waiting for network idle after 1s'); + + await vi.advanceTimersByTimeAsync(2000); + + await pending; + expect(mockDownloadArticle).not.toHaveBeenCalled(); + }); +}); + +describe('web fetch browser tier render-aware helpers', () => { + it('merges accessible same-origin iframe bodies into the extracted HTML', () => { + const dom = new JSDOM(` +
+

Shell

+ +
+ `, { url: 'https://example.com/main.html', runScripts: 'outside-only' }); + const frame = dom.window.document.querySelector('iframe')!; + frame.contentDocument!.open(); + frame.contentDocument!.write('
Name

    Station A 42

    '); + frame.contentDocument!.close(); + + const result = dom.window.eval(buildRenderAwareExtractorJs({ frames: 'same-origin' })) as ExtractedPage; + + expect(result.diagnostics.includedFrameCount).toBe(1); + expect(result.contentHtml).toContain('data-webcmd-iframe-source="https://example.com/frame.html"'); + expect(result.contentHtml).toContain('localized text iframe: https://example.com/frame.html'); + expect(result.contentHtml).toContain('Station A 42'); + expect(result.diagnostics.emptyContainers).toEqual(expect.arrayContaining([ + expect.objectContaining({ scope: 'iframe', id: 'gridDatas', url: 'https://example.com/frame.html' }), + ])); + expect(result.diagnostics.emptyContainers.every((item: { scope: string }) => item.scope === 'iframe')).toBe(true); + }); + + it('merges readable same-origin iframes outside the selected content element', () => { + const dom = new JSDOM(` +
    +

    Main Article

    +

    ${'Main content '.repeat(30)}

    +
    + + `, { url: 'https://example.com/main.html', runScripts: 'outside-only' }); + const frame = dom.window.document.querySelector('iframe')!; + frame.contentDocument!.open(); + frame.contentDocument!.write(`

    Outside Frame

    ${'Frame data '.repeat(12)}

    `); + frame.contentDocument!.close(); + + const result = dom.window.eval(buildRenderAwareExtractorJs({ frames: 'same-origin' })) as ExtractedPage; + + expect(result.diagnostics.includedFrameCount).toBe(1); + expect(result.contentHtml).toContain('data-webcmd-iframe-source="https://example.com/outside.html"'); + expect(result.contentHtml).toContain('Outside Frame'); + expect(result.contentHtml).toContain('Frame data'); + }); + + it('keeps short data-like iframes outside the selected content element', () => { + const dom = new JSDOM(` +
    +

    Main Article

    +

    ${'Main content '.repeat(30)}

    +
    + + `, { url: 'https://example.com/main.html', runScripts: 'outside-only' }); + const frame = dom.window.document.querySelector('iframe')!; + frame.contentDocument!.open(); + frame.contentDocument!.write('
    water level
    42
      '); + frame.contentDocument!.close(); + + const result = dom.window.eval(buildRenderAwareExtractorJs({ frames: 'same-origin' })) as ExtractedPage; + + expect(result.diagnostics.includedFrameCount).toBe(1); + expect(result.contentHtml).toContain('42'); + expect(result.diagnostics.emptyContainers).toEqual(expect.arrayContaining([ + expect.objectContaining({ scope: 'iframe', id: 'gridDatas', url: 'https://example.com/data.html' }), + ])); + expect(result.diagnostics.emptyContainers.every((item: { scope: string }) => item.scope === 'iframe')).toBe(true); + }); + + it('skips short non-structural iframes outside the selected content element', () => { + const dom = new JSDOM(` +
      +

      Main Article

      +

      ${'Main content '.repeat(30)}

      +
      + + `, { url: 'https://example.com/main.html', runScripts: 'outside-only' }); + const frame = dom.window.document.querySelector('iframe')!; + frame.contentDocument!.open(); + frame.contentDocument!.write('

      tiny note

      '); + frame.contentDocument!.close(); + + const result = dom.window.eval(buildRenderAwareExtractorJs({ frames: 'same-origin' })) as ExtractedPage; + + expect(result.diagnostics.includedFrameCount).toBe(0); + expect(result.contentHtml).not.toContain('tiny note'); + }); + + it('includes short non-structural iframes in all-same-origin mode', () => { + const dom = new JSDOM(` +
      +

      Main Article

      +

      ${'Main content '.repeat(30)}

      +
      + + `, { url: 'https://example.com/main.html', runScripts: 'outside-only' }); + const frame = dom.window.document.querySelector('iframe')!; + frame.contentDocument!.open(); + frame.contentDocument!.write('
      Online: 42°C
      '); + frame.contentDocument!.close(); + + const result = dom.window.eval(buildRenderAwareExtractorJs({ frames: 'all-same-origin' })) as ExtractedPage; + + expect(result.diagnostics.includedFrameCount).toBe(1); + expect(result.contentHtml).toContain('Online: 42°C'); + }); + + it('marks API-like network entries as interesting and ignores static assets', () => { + expect(isInterestingNetworkEntry({ + method: 'POST', + url: 'https://example.com/GJZ/Ajax/Publish.ashx', + status: 200, + contentType: 'text/html', + size: 100, + bodyTruncated: false, + })).toBe(true); + expect(isInterestingNetworkEntry({ + method: 'POST', + url: 'https://example.com/GJZ/Ajax/Publish.ashx', + status: 200, + contentType: 'application/json', + size: 100, + bodyTruncated: false, + })).toBe(true); + expect(isInterestingNetworkEntry({ + method: 'GET', + url: 'https://example.com/app.js', + status: 200, + contentType: 'application/javascript', + size: 100, + bodyTruncated: false, + })).toBe(false); + }); + + it('formats frame and XHR diagnostics for shell pages', () => { + const output = formatDiagnostics({ + diagnostics: { + url: 'https://example.com/main.html', + includedFrameCount: 1, + frames: [{ + index: 0, + src: 'https://example.com/frame.html', + title: '', + sameOrigin: true, + accessible: true, + textLength: 42, + }], + emptyContainers: [{ + scope: 'iframe', + url: 'https://example.com/frame.html', + tag: 'ul', + id: 'gridDatas', + className: '', + }], + }, + }, [{ + method: 'POST', + url: 'https://example.com/GJZ/Ajax/Publish.ashx', + status: 200, + contentType: 'application/json', + size: 64, + bodyTruncated: false, + }], true); + + expect(output).toContain('frames: 1, included_same_origin: 1'); + expect(output).toContain('[frame 0] same-origin accessible text=42 https://example.com/frame.html'); + expect(output).toContain('iframe: ul#gridDatas (https://example.com/frame.html)'); + expect(output).toContain('POST 200 application/json https://example.com/GJZ/Ajax/Publish.ashx'); + }); + + it('normalizes unknown frame and wait-until values to their defaults', () => { + expect(normalizeFrameMode('bogus')).toBe('same-origin'); + expect(normalizeFrameMode('none')).toBe('none'); + expect(normalizeWaitUntil('bogus')).toBe('domstable'); + expect(normalizeWaitUntil('networkidle')).toBe('networkidle'); + }); +}); diff --git a/clis/web/fetch-browser.js b/src/fetch/browser.ts similarity index 53% rename from clis/web/fetch-browser.js rename to src/fetch/browser.ts index c620a37e..ccbc7c10 100644 --- a/clis/web/fetch-browser.js +++ b/src/fetch/browser.ts @@ -1,7 +1,8 @@ /** - * Generic web page reader — fetch any URL and export as Markdown. + * Browser tier of `webcmd web fetch`. * - * Uses browser-side DOM heuristics to extract the main content: + * Renders a page in a real browser and extracts the main content with DOM + * heuristics: * 1.
      element * 2. [role="main"] element * 3.
      element @@ -9,102 +10,143 @@ * * Pipes through the shared article-download pipeline (Turndown + image download). * - * Usage: - * webcmd web fetch-browser --url "https://www.anthropic.com/research/..." --output ./articles - * webcmd web fetch-browser --url "https://..." --download-images false + * This is the escalation target when the plain and impit tiers in `client.ts` + * cannot read a page. It moved here from `clis/web/fetch-browser.js` so the + * whole fetch ladder ships in the core package rather than an adapter that was + * never installed by default (#247). */ -import { cli, Strategy } from '@agentrhq/webcmd/registry'; -import { downloadArticle } from '@agentrhq/webcmd/download/article-download'; +import { articleHtmlToMarkdown, downloadArticle } from '../download/article-download.js'; +import type { CommandArgs } from '../registry.js'; +import type { IPage } from '../types.js'; + +export const DEFAULT_OUTPUT_DIR = './web-articles'; const NETWORK_IDLE_QUIET_MS = 1000; const NETWORK_IDLE_POLL_MS = 500; const MIN_NON_STRUCTURAL_IFRAME_TEXT_CHARS = 50; -function sleep(ms) { - return new Promise(resolve => setTimeout(resolve, ms)); +export type FrameMode = 'same-origin' | 'all-same-origin' | 'none'; +export type WaitUntil = 'domstable' | 'networkidle'; + +export interface NetworkEntry { + method: string; + url: string; + status: number; + contentType: string; + size: number; + bodyTruncated: boolean; } -function boolish(value) { - if (value === true) return true; - if (typeof value === 'string') return ['1', 'true', 'yes', 'on'].includes(value.toLowerCase()); - return false; +export interface FrameDiagnostic { + index: number; + src: string; + title: string; + sameOrigin: boolean; + accessible: boolean; + textLength: number; } -function normalizeFrameMode(value) { - const mode = String(value || 'same-origin').toLowerCase(); - if (['same-origin', 'all-same-origin', 'none'].includes(mode)) return mode; - return 'same-origin'; +export interface ExtractedPage { + title: string; + author: string; + publishTime: string; + contentHtml: string; + imageUrls: string[]; + diagnostics: { + url: string; + frames: FrameDiagnostic[]; + emptyContainers: Array<{ scope: string; url: string; tag: string; id: string; className: string }>; + includedFrameCount: number; + }; } -function normalizeWaitUntil(value) { - const waitUntil = String(value || 'domstable').toLowerCase(); - if (['domstable', 'networkidle'].includes(waitUntil)) return waitUntil; - return 'domstable'; +function sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); } -function normalizeNetworkEntry(entry) { - const preview = typeof entry?.responsePreview === 'string' ? entry.responsePreview : ''; - return { - method: typeof entry?.method === 'string' ? entry.method : 'GET', - url: typeof entry?.url === 'string' ? entry.url : '', - status: typeof entry?.responseStatus === 'number' ? entry.responseStatus : 0, - contentType: typeof entry?.responseContentType === 'string' ? entry.responseContentType : '', - size: typeof entry?.responseBodyFullSize === 'number' ? entry.responseBodyFullSize : preview.length, - bodyTruncated: entry?.responseBodyTruncated === true, - }; +export function boolish(value: unknown): boolean { + if (value === true) return true; + if (typeof value === 'string') return ['1', 'true', 'yes', 'on'].includes(value.toLowerCase()); + return false; } -function isInterestingNetworkEntry(entry) { - const ct = (entry.contentType || '').toLowerCase(); - const url = entry.url || ''; - const method = (entry.method || 'GET').toUpperCase(); - const staticAsset = /\.(js|css|png|jpg|jpeg|gif|svg|woff|woff2|ico|map)(\?|$)/i.test(url); - const noisy = /analytics|tracking|telemetry|beacon|pixel|gtag|fbevents/i.test(url); - const apiLikeUrl = /\/(api|ajax|graphql|rest|service|handler)(\/|[?._-]|$)|\.(ashx|aspx|asmx|php)(\?|$)/i.test(url); - const dataLikeContent = ct.includes('json') - || ct.includes('xml') - || ct.includes('text/plain') - || ct.includes('javascript') - || (apiLikeUrl && ct.includes('text/html')); - return ( - !staticAsset - && !noisy - && (dataLikeContent || apiLikeUrl || method !== 'GET') - ); +export function normalizeFrameMode(value: unknown): FrameMode { + const mode = String(value || 'same-origin').toLowerCase(); + if (mode === 'same-origin' || mode === 'all-same-origin' || mode === 'none') return mode; + return 'same-origin'; } -async function drainNetworkCapture(page, sink) { - if (!page.readNetworkCapture) return []; - const raw = await page.readNetworkCapture().catch(() => []); - const entries = Array.isArray(raw) ? raw.map(normalizeNetworkEntry).filter(entry => entry.url) : []; - sink.push(...entries); - return entries; +export function normalizeWaitUntil(value: unknown): WaitUntil { + const waitUntil = String(value || 'domstable').toLowerCase(); + if (waitUntil === 'domstable' || waitUntil === 'networkidle') return waitUntil; + return 'domstable'; } -async function maybeStartNetworkCapture(page) { - if (!page.startNetworkCapture) return false; - try { - return await page.startNetworkCapture(''); - } catch { - return false; - } +function normalizeNetworkEntry(entry: Record | null | undefined): NetworkEntry { + const preview = typeof entry?.responsePreview === 'string' ? entry.responsePreview : ''; + return { + method: typeof entry?.method === 'string' ? entry.method : 'GET', + url: typeof entry?.url === 'string' ? entry.url : '', + status: typeof entry?.responseStatus === 'number' ? entry.responseStatus : 0, + contentType: typeof entry?.responseContentType === 'string' ? entry.responseContentType : '', + size: typeof entry?.responseBodyFullSize === 'number' ? entry.responseBodyFullSize : preview.length, + bodyTruncated: entry?.responseBodyTruncated === true, + }; } -async function waitForNetworkIdle(page, maxSeconds, sink) { - const timeoutMs = Math.max(1, Number(maxSeconds) || 1) * 1000; - const deadline = Date.now() + timeoutMs; - let quietSince = Date.now(); - while (Date.now() < deadline) { - const entries = await drainNetworkCapture(page, sink); - if (entries.length > 0) quietSince = Date.now(); - if (Date.now() - quietSince >= NETWORK_IDLE_QUIET_MS) return { ok: true }; - await sleep(NETWORK_IDLE_POLL_MS); - } - return { ok: false, timedOut: true }; +export function isInterestingNetworkEntry(entry: NetworkEntry): boolean { + const ct = (entry.contentType || '').toLowerCase(); + const url = entry.url || ''; + const method = (entry.method || 'GET').toUpperCase(); + const staticAsset = /\.(js|css|png|jpg|jpeg|gif|svg|woff|woff2|ico|map)(\?|$)/i.test(url); + const noisy = /analytics|tracking|telemetry|beacon|pixel|gtag|fbevents/i.test(url); + const apiLikeUrl = /\/(api|ajax|graphql|rest|service|handler)(\/|[?._-]|$)|\.(ashx|aspx|asmx|php)(\?|$)/i.test(url); + const dataLikeContent = ct.includes('json') + || ct.includes('xml') + || ct.includes('text/plain') + || ct.includes('javascript') + || (apiLikeUrl && ct.includes('text/html')); + return ( + !staticAsset + && !noisy + && (dataLikeContent || apiLikeUrl || method !== 'GET') + ); +} + +async function drainNetworkCapture(page: IPage, sink: NetworkEntry[]): Promise { + if (!page.readNetworkCapture) return []; + const raw = await page.readNetworkCapture().catch(() => []); + const entries = Array.isArray(raw) + ? raw.map(entry => normalizeNetworkEntry(entry as Record)).filter(entry => entry.url) + : []; + sink.push(...entries); + return entries; +} + +async function maybeStartNetworkCapture(page: IPage): Promise { + if (!page.startNetworkCapture) return false; + try { + return await page.startNetworkCapture(''); + } catch { + return false; + } } -function buildWaitForSelectorAcrossFramesJs(selector, timeoutMs) { - return ` +async function waitForNetworkIdle(page: IPage, maxSeconds: number, sink: NetworkEntry[]): Promise<{ ok: boolean; timedOut?: boolean }> { + const timeoutMs = Math.max(1, Number(maxSeconds) || 1) * 1000; + const deadline = Date.now() + timeoutMs; + let quietSince = Date.now(); + while (Date.now() < deadline) { + const entries = await drainNetworkCapture(page, sink); + if (entries.length > 0) quietSince = Date.now(); + if (Date.now() - quietSince >= NETWORK_IDLE_QUIET_MS) return { ok: true }; + await sleep(NETWORK_IDLE_POLL_MS); + } + return { ok: false, timedOut: true }; +} + +export function buildWaitForSelectorAcrossFramesJs(selector: string, timeoutMs: number): string { + return ` (async () => { const selector = ${JSON.stringify(selector)}; const timeoutAt = Date.now() + ${Number(timeoutMs) || 10000}; @@ -141,8 +183,8 @@ function buildWaitForSelectorAcrossFramesJs(selector, timeoutMs) { `; } -function buildRenderAwareExtractorJs(options) { - return ` +export function buildRenderAwareExtractorJs(options: { frames: FrameMode }): string { + return ` (() => { const frameMode = ${JSON.stringify(options.frames)}; const minNonStructuralIframeTextChars = ${MIN_NON_STRUCTURAL_IFRAME_TEXT_CHARS}; @@ -368,124 +410,144 @@ function buildRenderAwareExtractorJs(options) { `; } -function formatDiagnostics(data, networkEntries, captureSupported) { - const lines = []; - const diag = data?.diagnostics || {}; - lines.push('[web-fetch-browser diagnose]'); - lines.push(`url: ${diag.url || '-'}`); - lines.push(`frames: ${Array.isArray(diag.frames) ? diag.frames.length : 0}, included_same_origin: ${diag.includedFrameCount || 0}`); - for (const frame of (diag.frames || []).slice(0, 20)) { - lines.push(` [frame ${frame.index}] ${frame.sameOrigin ? 'same-origin' : 'cross-origin'} ${frame.accessible ? 'accessible' : 'blocked'} text=${frame.textLength || 0} ${frame.src || '-'}`); - } - if (Array.isArray(diag.emptyContainers) && diag.emptyContainers.length > 0) { - lines.push(`empty_containers: ${diag.emptyContainers.length}`); - for (const item of diag.emptyContainers.slice(0, 12)) { - const selector = `${item.tag}${item.id ? `#${item.id}` : ''}${item.className ? `.${String(item.className).trim().split(/\\s+/).filter(Boolean).join('.')}` : ''}`; - lines.push(` ${item.scope}: ${selector} (${item.url || '-'})`); - } - } - const interesting = networkEntries.filter(isInterestingNetworkEntry); - lines.push(`network_capture: ${captureSupported ? 'enabled' : 'unavailable'}, entries=${networkEntries.length}, api_like=${interesting.length}`); - for (const entry of interesting.slice(0, 20)) { - lines.push(` ${entry.method} ${entry.status || '-'} ${entry.contentType || '-'} ${entry.url}`); +/** + * A page that navigates after load — a challenge interstitial handing off to + * the real page, a client-side redirect — destroys the execution context out + * from under `evaluate`. That is the normal case on the escalation path, since + * escalation only happens for pages that blocked plain HTTP in the first place. + * Retry once after letting the new document settle; a second failure is real. + */ +async function evaluateAfterNavigation(page: IPage, js: string, settleSeconds: number): Promise { + try { + return await page.evaluate(js); + } catch (error) { + if (!/execution context was destroyed|context was destroyed|navigation/i.test(String((error as Error)?.message ?? error))) throw error; + await page.wait(Math.max(1, settleSeconds)); + return page.evaluate(js); + } +} + +export function formatDiagnostics( + data: Partial | null | undefined, + networkEntries: NetworkEntry[], + captureSupported: boolean, +): string { + const lines: string[] = []; + const diag = data?.diagnostics ?? ({} as Partial); + lines.push('[web-fetch-browser diagnose]'); + lines.push(`url: ${diag.url || '-'}`); + lines.push(`frames: ${Array.isArray(diag.frames) ? diag.frames.length : 0}, included_same_origin: ${diag.includedFrameCount || 0}`); + for (const frame of (diag.frames || []).slice(0, 20)) { + lines.push(` [frame ${frame.index}] ${frame.sameOrigin ? 'same-origin' : 'cross-origin'} ${frame.accessible ? 'accessible' : 'blocked'} text=${frame.textLength || 0} ${frame.src || '-'}`); + } + if (Array.isArray(diag.emptyContainers) && diag.emptyContainers.length > 0) { + lines.push(`empty_containers: ${diag.emptyContainers.length}`); + for (const item of diag.emptyContainers.slice(0, 12)) { + const selector = `${item.tag}${item.id ? `#${item.id}` : ''}${item.className ? `.${String(item.className).trim().split(/\s+/).filter(Boolean).join('.')}` : ''}`; + lines.push(` ${item.scope}: ${selector} (${item.url || '-'})`); } - return `${lines.join('\n')}\n`; + } + const interesting = networkEntries.filter(isInterestingNetworkEntry); + lines.push(`network_capture: ${captureSupported ? 'enabled' : 'unavailable'}, entries=${networkEntries.length}, api_like=${interesting.length}`); + for (const entry of interesting.slice(0, 20)) { + lines.push(` ${entry.method} ${entry.status || '-'} ${entry.contentType || '-'} ${entry.url}`); + } + return `${lines.join('\n')}\n`; } -const command = cli({ - site: 'web', - name: 'fetch-browser', - access: 'read', - description: 'Fetch any web page and export as Markdown', - strategy: Strategy.COOKIE, - navigateBefore: false, // we handle navigation ourselves - args: [ - { name: 'url', required: true, help: 'Any web page URL' }, - { name: 'output', default: './web-articles', help: 'Output directory' }, - { name: 'download-images', type: 'boolean', default: true, help: 'Download images locally' }, - { name: 'wait', type: 'int', default: 3, help: 'Seconds to wait after page load' }, - { name: 'wait-for', valueRequired: true, help: 'CSS selector to wait for in the main document or same-origin iframes' }, - { name: 'wait-until', default: 'domstable', choices: ['domstable', 'networkidle'], help: 'Readiness policy after navigation: domstable or networkidle' }, - { name: 'frames', default: 'same-origin', choices: ['same-origin', 'all-same-origin', 'none'], help: 'Iframe handling mode: relevant same-origin, all-same-origin, or none' }, - { name: 'diagnose', type: 'boolean', default: false, help: 'Print render diagnostics (frames, empty containers, XHR/API-like requests) to stderr' }, - { name: 'stdout', type: 'boolean', default: false, help: 'Print markdown to stdout instead of saving to a file' }, - ], - columns: ['title', 'author', 'publish_time', 'status', 'size', 'saved'], - func: async (page, kwargs, debug = false) => { - const url = kwargs.url; - const waitSeconds = kwargs.wait ?? 3; - const waitUntil = normalizeWaitUntil(kwargs['wait-until']); - const frameMode = normalizeFrameMode(kwargs.frames); - const shouldDiagnose = boolish(kwargs.diagnose) || debug || !!process.env.WEBCMD_VERBOSE; - const networkEntries = []; - const captureSupported = (waitUntil === 'networkidle' || shouldDiagnose) - ? await maybeStartNetworkCapture(page) - : false; - // Navigate to the target URL - await page.goto(url); - if (kwargs['wait-for']) { - const waitResult = await page.evaluate(buildWaitForSelectorAcrossFramesJs(String(kwargs['wait-for']), waitSeconds * 1000)); - if (waitResult?.invalidSelector) { - throw new Error(`Invalid --wait-for selector "${kwargs['wait-for']}": ${waitResult.error || 'querySelector failed'}`); - } - if (!waitResult?.ok) { - throw new Error(`Timed out waiting for selector "${kwargs['wait-for']}" in main document or same-origin iframes`); - } - } else if (waitUntil !== 'networkidle') { - await page.wait(waitSeconds); - } - if (waitUntil === 'networkidle') { - if (!captureSupported) { - throw new Error('Network capture is unavailable, so --wait-until networkidle cannot be satisfied'); - } - const idle = await waitForNetworkIdle(page, waitSeconds, networkEntries); - if (!idle?.ok) { - throw new Error(`Timed out waiting for network idle after ${waitSeconds}s`); - } - } - // Extract article content using browser-side heuristics - const data = await page.evaluate(buildRenderAwareExtractorJs({ frames: frameMode })); - if (captureSupported) await drainNetworkCapture(page, networkEntries); - if (shouldDiagnose) process.stderr.write(formatDiagnostics(data, networkEntries, captureSupported)); - // Determine Referer from URL for image downloads - let referer = ''; - try { - const parsed = new URL(url); - referer = parsed.origin + '/'; - } - catch { /* ignore */ } - const result = await downloadArticle({ - title: data?.title || 'untitled', - author: data?.author, - publishTime: data?.publishTime, - sourceUrl: url, - contentHtml: data?.contentHtml || '', - imageUrls: data?.imageUrls, - }, { - output: kwargs.output, - downloadImages: kwargs['download-images'], - imageHeaders: referer ? { Referer: referer } : undefined, - stdout: kwargs.stdout, - configureTurndown: (td) => { - td.addRule('preserveButtons', { - filter: (node) => node.nodeName === 'BUTTON', - replacement: (content) => content, - }); - }, - }); - // `--stdout` is a content-streaming mode. The markdown body already went - // to process.stdout inside downloadArticle(), so returning rows here - // would make Commander append table/JSON output to the same stdout - // stream and break piping. - return kwargs.stdout ? null : result; +/** + * Render `--url` in the browser and return its main content as markdown. + * + * This is the escalation target for `web fetch`: it deliberately writes no + * files and downloads no images, so an escalated fetch answers in the same + * shape as the plain and impit tiers and still honours `-f`. The file-export + * pipeline stays behind `web fetch-browser`. + */ +export async function extractPageMarkdown(page: IPage, kwargs: CommandArgs): Promise<{ title: string; content: string }> { + const url = String(kwargs.url); + const waitSeconds = Number(kwargs.wait ?? 3); + await page.goto(url); + await page.wait(waitSeconds); + const data = await evaluateAfterNavigation( + page, + buildRenderAwareExtractorJs({ frames: normalizeFrameMode(kwargs.frames) }), + waitSeconds, + ); + return { + title: data?.title || '', + content: articleHtmlToMarkdown(data?.contentHtml || ''), + }; +} + +/** + * Render `--url` in the browser and hand the extracted article to the shared + * download pipeline. Returns `null` in `--stdout` mode: the markdown body has + * already gone to process.stdout inside downloadArticle(), so returning rows + * would make Commander append table/JSON output to the same stream and break + * piping. + */ +export async function runFetchBrowser(page: IPage, kwargs: CommandArgs, debug: boolean = false): Promise { + const url = String(kwargs.url); + const waitSeconds = Number(kwargs.wait ?? 3); + const waitUntil = normalizeWaitUntil(kwargs['wait-until']); + const frameMode = normalizeFrameMode(kwargs.frames); + const shouldDiagnose = boolish(kwargs.diagnose) || debug || !!process.env.WEBCMD_VERBOSE; + const networkEntries: NetworkEntry[] = []; + const captureSupported = (waitUntil === 'networkidle' || shouldDiagnose) + ? await maybeStartNetworkCapture(page) + : false; + // Navigate to the target URL + await page.goto(url); + if (kwargs['wait-for']) { + const waitResult = await page.evaluate<{ ok?: boolean; invalidSelector?: boolean; error?: string }>( + buildWaitForSelectorAcrossFramesJs(String(kwargs['wait-for']), waitSeconds * 1000), + ); + if (waitResult?.invalidSelector) { + throw new Error(`Invalid --wait-for selector "${kwargs['wait-for']}": ${waitResult.error || 'querySelector failed'}`); + } + if (!waitResult?.ok) { + throw new Error(`Timed out waiting for selector "${kwargs['wait-for']}" in main document or same-origin iframes`); + } + } else if (waitUntil !== 'networkidle') { + await page.wait(waitSeconds); + } + if (waitUntil === 'networkidle') { + if (!captureSupported) { + throw new Error('Network capture is unavailable, so --wait-until networkidle cannot be satisfied'); + } + const idle = await waitForNetworkIdle(page, waitSeconds, networkEntries); + if (!idle?.ok) { + throw new Error(`Timed out waiting for network idle after ${waitSeconds}s`); + } + } + // Extract article content using browser-side heuristics + const data = await evaluateAfterNavigation(page, buildRenderAwareExtractorJs({ frames: frameMode }), waitSeconds); + if (captureSupported) await drainNetworkCapture(page, networkEntries); + if (shouldDiagnose) process.stderr.write(formatDiagnostics(data, networkEntries, captureSupported)); + // Determine Referer from URL for image downloads + let referer = ''; + try { + const parsed = new URL(url); + referer = `${parsed.origin}/`; + } catch { /* ignore */ } + const result = await downloadArticle({ + title: data?.title || 'untitled', + author: data?.author, + publishTime: data?.publishTime, + sourceUrl: url, + contentHtml: data?.contentHtml || '', + imageUrls: data?.imageUrls, + }, { + output: String(kwargs.output ?? DEFAULT_OUTPUT_DIR), + downloadImages: kwargs['download-images'] !== false, + imageHeaders: referer ? { Referer: referer } : undefined, + stdout: kwargs.stdout === true, + configureTurndown: (td) => { + td.addRule('preserveButtons', { + filter: (node) => node.nodeName === 'BUTTON', + replacement: (content) => content, + }); }, -}); -export const __test__ = { - command, - buildRenderAwareExtractorJs, - buildWaitForSelectorAcrossFramesJs, - formatDiagnostics, - isInterestingNetworkEntry, - normalizeFrameMode, - normalizeWaitUntil, -}; + }); + return kwargs.stdout ? null : result; +} diff --git a/src/fetch/classify.test.ts b/src/fetch/classify.test.ts index 7d4f42f7..ebf3b128 100644 --- a/src/fetch/classify.test.ts +++ b/src/fetch/classify.test.ts @@ -6,5 +6,59 @@ describe('fetch classification', () => { expect(isChallengeResponse(403, { server: 'cloudflare' }, 'Just a moment...')).toBe(true); expect(isChallengeResponse(403, {}, 'forbidden')).toBe(false); }); + it('recognizes script-heavy app shells', () => expect(isJavaScriptShell('
      ')).toBe(true)); + + // A CSP allow-list names third parties this page may load; it says nothing + // about who served the response. news.ycombinator.com returns a healthy 200 + // whose CSP mentions cdnjs.cloudflare.com and google.com/recaptcha (#264). + it('ignores content-security-policy allow-lists naming a CDN or captcha vendor', () => { + const csp = "default-src 'self'; script-src 'self' https://www.google.com/recaptcha/ https://cdnjs.cloudflare.com/"; + expect(isChallengeResponse(200, { 'content-security-policy': csp }, 'Hacker News')).toBe(false); + }); + + it('ignores report-to and link headers naming a challenge vendor', () => { + const headers = { 'report-to': '{"endpoints":[{"url":"https://report.cloudflare.com/"}]}', link: '; rel=preconnect' }; + expect(isChallengeResponse(200, headers, 'Real content')).toBe(false); + }); + + // example.com is a plain static page fronted by Cloudflare. `server: cloudflare` + // on a 200 that was actually served is not a challenge (#283). + it('does not flag a Cloudflare-fronted 200 whose body is a real page', () => { + const headers = { server: 'cloudflare', 'cf-cache-status': 'HIT', 'content-type': 'text/html' }; + expect(isChallengeResponse(200, headers, '

      Example Domain

      ')).toBe(false); + }); + + it('still flags a managed-challenge interstitial served with a 200', () => { + const headers = { server: 'cloudflare', 'cf-mitigated': 'challenge' }; + expect(isChallengeResponse(200, headers, 'Just a moment...')).toBe(true); + }); + + // cf-mitigated only ever appears on an actual mitigation, so it stands alone + // even when the body has been withheld. + it('treats challenge-specific headers as evidence without a body marker', () => { + expect(isChallengeResponse(403, { 'cf-mitigated': 'challenge' }, '')).toBe(true); + expect(isChallengeResponse(429, { 'x-datadome': 'protected' }, '')).toBe(true); + expect(isChallengeResponse(503, { 'cf-chl-bypass': '1' }, '')).toBe(true); + }); + + it('treats a challenge cookie as evidence', () => { + expect(isChallengeResponse(403, { 'set-cookie': '__cf_bm=abc; Path=/' }, 'blocked')).toBe(true); + expect(isChallengeResponse(200, { 'set-cookie': 'session=abc; Path=/' }, 'real page')).toBe(false); + }); + + // A bare CDN name corroborates an already-suspicious status, but never decides + // on its own — that is the difference between #283 and a genuine block. + it('lets a bare CDN name corroborate a non-200 block', () => { + expect(isChallengeResponse(403, { server: 'cloudflare' }, 'Access denied')).toBe(true); + expect(isChallengeResponse(200, { server: 'akamai' }, 'Real page content')).toBe(false); + }); + + it('does not flag a healthy 200 with no challenge evidence anywhere', () => { + expect(isChallengeResponse(200, { 'content-type': 'text/html' }, 'Hello')).toBe(false); + }); + + it('still catches a body-level captcha wall on a 403', () => { + expect(isChallengeResponse(403, {}, '
      verify you are human
      ')).toBe(true); + }); }); diff --git a/src/fetch/classify.ts b/src/fetch/classify.ts index 4051c13c..ec7d08cf 100644 --- a/src/fetch/classify.ts +++ b/src/fetch/classify.ts @@ -1,8 +1,37 @@ -const challengeMarkers = /cloudflare|cf-chl|datadome|perimeterx|px-captcha|akamai|captcha|just a moment|verify you are human/i; +/** + * Headers that describe *this* response. Everything else is excluded on + * purpose: `content-security-policy`, `report-to` and `link` are allow-lists of + * third parties a page may load, so a CSP naming `cdnjs.cloudflare.com` or + * `google.com/recaptcha` proves nothing about who served the bytes (#264). + */ +const CHALLENGE_HEADERS = /^(?:server|cf-mitigated|cf-chl-[\w-]+|x-datadome[\w-]*|set-cookie)$/i; + +/** + * Markers that do not legitimately appear outside an actual challenge, so they + * decide on their own at any status — including the managed-challenge + * interstitial Cloudflare serves with a 200. + */ +const DECISIVE_MARKERS = /cf-chl|cf-mitigated|__cf_bm|datadome|perimeterx|px-captcha|just a moment|verify you are human|checking your browser|enable javascript and cookies/i; + +/** + * Markers that appear constantly on healthy pages: a CDN name in `server:`, + * or a reCAPTCHA widget embedded in an ordinary login form. These only + * corroborate a status that already looks like a block, never decide alone — + * that is the difference between a real block and the false positive in #283. + */ +const CORROBORATING_MARKERS = /cloudflare|akamai|captcha/i; + +const BLOCKED_STATUSES = new Set([403, 429, 503]); export function isChallengeResponse(status: number, headers: Record, body: string): boolean { - const evidence = `${Object.entries(headers).map(([key, value]) => `${key}:${value}`).join('\n')}\n${body.slice(0, 20_000)}`; - return challengeMarkers.test(evidence) && (status === 403 || status === 429 || status === 503 || status === 200); + const evidence = [ + ...Object.entries(headers) + .filter(([key]) => CHALLENGE_HEADERS.test(key)) + .map(([key, value]) => `${key}:${value}`), + body.slice(0, 20_000), + ].join('\n'); + if (DECISIVE_MARKERS.test(evidence)) return true; + return BLOCKED_STATUSES.has(status) && CORROBORATING_MARKERS.test(evidence); } export function isJavaScriptShell(body: string): boolean { diff --git a/src/fetch/client.ts b/src/fetch/client.ts index 07d701af..f42da73b 100644 --- a/src/fetch/client.ts +++ b/src/fetch/client.ts @@ -7,8 +7,8 @@ import { isChallengeResponse, isJavaScriptShell } from './classify.js'; export interface WebFetchOptions { url: string; timeoutSeconds: number; maxChars: number; allowPrivate: boolean; } export interface WebFetchResult { - status: number; requestedUrl: string; finalUrl: string; contentType: string; tier: 'plain' | 'impit'; profile?: 'chrome' | 'firefox'; - title: string; extractionSource: ExtractFetchedContentResult['source']; truncated: boolean; content: string; + status: number; requestedUrl: string; finalUrl: string; contentType: string; tier: 'plain' | 'impit' | 'browser'; profile?: 'chrome' | 'firefox'; + title: string; extractionSource: ExtractFetchedContentResult['source'] | 'browser'; truncated: boolean; content: string; } type ResponseLike = Pick & { body?: ReadableStream | null; bytes?: () => Promise; }; type FetchLike = (url: string, options?: Record) => Promise; @@ -49,7 +49,7 @@ export async function webFetch(options: WebFetchOptions, dependencies: WebFetchD let response = await plainFetch(options.url, { redirect: 'manual', dispatcher: new ProxyAgent(proxy.url), signal: AbortSignal.timeout(remaining()) }); let body = await readBody(response); let tier: WebFetchResult['tier'] = 'plain'; let profile: WebFetchResult['profile']; - if (isJavaScriptShell(body)) throw new CliError('FETCH_REQUIRES_BROWSER', 'This page requires browser rendering.', 'Use webcmd web fetch-browser for this URL.'); + if (isJavaScriptShell(body)) throw new CliError('FETCH_REQUIRES_BROWSER', 'This page requires browser rendering.', 'The browser tier renders this page; web fetch escalates to it automatically.'); if (isChallengeResponse(response.status, headersOf(response), body)) { // ponytail: impit's timeout covers the request, not the body stream, so a // trickling escalation body can outlive the budget. Race readBody against @@ -58,10 +58,10 @@ export async function webFetch(options: WebFetchOptions, dependencies: WebFetchD const impit = createImpit({ browser, proxyUrl: proxy.url, timeout: remaining() }); response = await impit.fetch(options.url, { redirect: 'manual', timeout: remaining() }); body = await readBody(response); tier = 'impit'; profile = browser; - if (isJavaScriptShell(body)) throw new CliError('FETCH_REQUIRES_BROWSER', 'This page requires browser rendering.', 'Use webcmd web fetch-browser for this URL.'); + if (isJavaScriptShell(body)) throw new CliError('FETCH_REQUIRES_BROWSER', 'This page requires browser rendering.', 'The browser tier renders this page; web fetch escalates to it automatically.'); if (!isChallengeResponse(response.status, headersOf(response), body)) break; } - if (isChallengeResponse(response.status, headersOf(response), body)) throw new CliError('FETCH_BLOCKED', 'The site blocked non-browser fetches.', 'Use webcmd web fetch-browser for this URL.'); + if (isChallengeResponse(response.status, headersOf(response), body)) throw new CliError('FETCH_BLOCKED', 'The site blocked non-browser fetches.', 'The browser tier renders this page; web fetch escalates to it automatically.'); } const extracted = extractFetchedContent({ body, contentType: response.headers.get('content-type') ?? '', url: options.url }); const clipped = truncate(extracted.content, options.maxChars); diff --git a/src/fetch/command.test.ts b/src/fetch/command.test.ts index 5b9823d2..d753a73d 100644 --- a/src/fetch/command.test.ts +++ b/src/fetch/command.test.ts @@ -1,13 +1,112 @@ import { describe, expect, it, vi } from 'vitest'; -import { formatWebFetchMarkdown, runClientOwnedWebFetch } from './command.js'; +import { CliError, TimeoutError } from '../errors.js'; +import { formatWebFetchMarkdown, runClientOwnedWebFetch, webFetchBrowserCommand, webFetchCommand } from './command.js'; + +const { mockExecuteCommand } = vi.hoisted(() => ({ mockExecuteCommand: vi.fn() })); +vi.mock('../execution.js', () => ({ executeCommand: mockExecuteCommand, prepareCommandArgs: (_cmd: unknown, k: unknown) => k })); + +const plainResult = { status: 200, requestedUrl: 'https://a', finalUrl: 'https://a', contentType: 'text/plain', tier: 'plain' as const, title: '', extractionSource: 'raw' as const, truncated: false, content: 'ok' }; describe('web fetch command', () => { it('renders fetch metadata before content', () => { - expect(formatWebFetchMarkdown({ status: 200, requestedUrl: 'https://a', finalUrl: 'https://b', contentType: 'text/plain', tier: 'plain', title: 'T', extractionSource: 'raw', truncated: false, content: 'body' })).toContain('Source: https://a'); + expect(formatWebFetchMarkdown({ ...plainResult, requestedUrl: 'https://a', finalUrl: 'https://b', title: 'T', content: 'body' })).toContain('Source: https://a'); }); + it('runs the client-owned command without Cloud routing', async () => { - const webFetch = vi.fn().mockResolvedValue({ status: 200, requestedUrl: 'https://a', finalUrl: 'https://a', contentType: 'text/plain', tier: 'plain', title: '', extractionSource: 'raw', truncated: false, content: 'ok' }); + const webFetch = vi.fn().mockResolvedValue(plainResult); await runClientOwnedWebFetch(['web', 'fetch', '--url', 'https://a'], { webFetch, stdout: { write: vi.fn() } as never }); expect(webFetch).toHaveBeenCalledOnce(); }); + + it('registers both tiers under the web site', () => { + expect(webFetchCommand.site).toBe('web'); + expect(webFetchCommand.browser).toBe(false); + expect(webFetchBrowserCommand.name).toBe('fetch-browser'); + expect(webFetchBrowserCommand.browser).toBe(true); + }); +}); + +describe('web fetch browser escalation', () => { + const run = (kwargs: Record) => (webFetchCommand.func as (k: unknown, d?: boolean) => Promise)(kwargs); + + it('escalates to the browser tier when the site blocks plain HTTP', async () => { + mockExecuteCommand.mockReset().mockResolvedValue({ title: 'Real Title', content: '# rendered' }); + const blocked = new CliError('FETCH_BLOCKED', 'The site blocked non-browser fetches.'); + vi.spyOn(await import('./client.js'), 'webFetch').mockRejectedValueOnce(blocked); + + const result = await run({ url: 'https://blocked.example', timeout: 30, 'max-chars': 50000, browser: true }); + + expect(mockExecuteCommand).toHaveBeenCalledOnce(); + expect(result).toMatchObject({ tier: 'browser', title: 'Real Title', content: '# rendered', extractionSource: 'browser' }); + }); + + it('escalates when the page needs browser rendering', async () => { + mockExecuteCommand.mockReset().mockResolvedValue({ title: 'App', content: 'shell content' }); + const needsBrowser = new CliError('FETCH_REQUIRES_BROWSER', 'This page requires browser rendering.'); + vi.spyOn(await import('./client.js'), 'webFetch').mockRejectedValueOnce(needsBrowser); + + const result = await run({ url: 'https://spa.example', browser: true }); + + expect(result).toMatchObject({ tier: 'browser', content: 'shell content' }); + }); + + // --browser false is the opt-out for callers that must never launch a browser. + it('rethrows instead of escalating when --browser false is given', async () => { + mockExecuteCommand.mockReset(); + const blocked = new CliError('FETCH_BLOCKED', 'The site blocked non-browser fetches.'); + vi.spyOn(await import('./client.js'), 'webFetch').mockRejectedValueOnce(blocked); + + await expect(run({ url: 'https://blocked.example', browser: false })).rejects.toThrow('The site blocked non-browser fetches.'); + expect(mockExecuteCommand).not.toHaveBeenCalled(); + }); + + // The stock hint tells the caller the browser tier runs automatically. When + // they turned it off, that hint is actively wrong — say what to do instead. + it('replaces the hint with the reason escalation was declined', async () => { + mockExecuteCommand.mockReset(); + vi.spyOn(await import('./client.js'), 'webFetch').mockRejectedValueOnce(new CliError('FETCH_BLOCKED', 'The site blocked non-browser fetches.', 'stock hint')); + + const error = await run({ url: 'https://blocked.example', browser: false }).catch((e: CliError) => e); + + expect(error).toBeInstanceOf(CliError); + expect((error as CliError).code).toBe('FETCH_BLOCKED'); + expect((error as CliError).hint).toContain('--browser false'); + }); + + // A timeout or a refused connection is a real failure. Escalating would hide + // a broken URL behind a slow browser run. + it('does not escalate a timeout', async () => { + mockExecuteCommand.mockReset(); + vi.spyOn(await import('./client.js'), 'webFetch').mockRejectedValueOnce(new TimeoutError('web fetch', 30)); + + await expect(run({ url: 'https://slow.example', browser: true })).rejects.toThrow(); + expect(mockExecuteCommand).not.toHaveBeenCalled(); + }); + + it('does not escalate an unrelated CliError', async () => { + mockExecuteCommand.mockReset(); + vi.spyOn(await import('./client.js'), 'webFetch').mockRejectedValueOnce(new CliError('FETCH_BODY_TOO_LARGE', 'Fetched body exceeds 10 MiB')); + + await expect(run({ url: 'https://big.example', browser: true })).rejects.toThrow('Fetched body exceeds 10 MiB'); + expect(mockExecuteCommand).not.toHaveBeenCalled(); + }); + + it('escalates on the client-owned fast path too', async () => { + mockExecuteCommand.mockReset().mockResolvedValue({ title: 'Rendered', content: 'browser body' }); + const webFetch = vi.fn().mockRejectedValue(new CliError('FETCH_BLOCKED', 'blocked')); + const write = vi.fn(); + + await runClientOwnedWebFetch(['web', 'fetch', '--url', 'https://blocked.example'], { webFetch, stdout: { write } as never }); + + expect(mockExecuteCommand).toHaveBeenCalledOnce(); + expect(write.mock.calls[0][0]).toContain('browser body'); + }); + + it('honours --browser false on the client-owned fast path', async () => { + mockExecuteCommand.mockReset(); + const webFetch = vi.fn().mockRejectedValue(new CliError('FETCH_BLOCKED', 'blocked')); + + await expect(runClientOwnedWebFetch(['web', 'fetch', '--url', 'https://blocked.example', '--browser', 'false'], { webFetch, stdout: { write: vi.fn() } as never })).rejects.toThrow('blocked'); + expect(mockExecuteCommand).not.toHaveBeenCalled(); + }); }); diff --git a/src/fetch/command.ts b/src/fetch/command.ts index 103e8487..d39c4b22 100644 --- a/src/fetch/command.ts +++ b/src/fetch/command.ts @@ -1,24 +1,130 @@ -import { cli, Strategy } from '../registry.js'; -import { ArgumentError } from '../errors.js'; -import { webFetch, type WebFetchOptions, type WebFetchResult } from './client.js'; +import { cli, Strategy, type CliCommand, type CommandArgs } from '../registry.js'; +import { ArgumentError, CliError } from '../errors.js'; +import type { webFetch, WebFetchOptions, WebFetchResult } from './client.js'; +import type { IPage } from '../types.js'; + +/** `./client.js` pulls in impit and undici. Both tiers import it lazily so a + * `webcmd ` startup never pays for the fetch stack. */ +const loadWebFetch = async (): Promise => (await import('./client.js')).webFetch; + +/** Kept in sync with `DEFAULT_OUTPUT_DIR` in ./browser.js, which is imported + * lazily so Turndown and the download pipeline stay out of CLI startup. */ +const DEFAULT_OUTPUT_DIR = './web-articles'; + +/** + * Failures the browser tier can still recover from. Anything else — a timeout, + * a refused connection, an oversized body — is a real error and is rethrown, so + * escalation never masks a broken URL as a slow one. + */ +const ESCALATION_CODES = new Set(['FETCH_BLOCKED', 'FETCH_REQUIRES_BROWSER']); + +export const webFetchBrowserCommand = cli({ + site: 'web', name: 'fetch-browser', access: 'read', + description: 'Fetch any web page in a real browser and export as Markdown', + strategy: Strategy.COOKIE, + navigateBefore: false, // we handle navigation ourselves + args: [ + { name: 'url', required: true, help: 'Any web page URL' }, + { name: 'output', default: DEFAULT_OUTPUT_DIR, help: 'Output directory' }, + { name: 'download-images', type: 'boolean', default: true, help: 'Download images locally' }, + { name: 'wait', type: 'int', default: 3, help: 'Seconds to wait after page load' }, + { name: 'wait-for', valueRequired: true, help: 'CSS selector to wait for in the main document or same-origin iframes' }, + { name: 'wait-until', default: 'domstable', choices: ['domstable', 'networkidle'], help: 'Readiness policy after navigation: domstable or networkidle' }, + { name: 'frames', default: 'same-origin', choices: ['same-origin', 'all-same-origin', 'none'], help: 'Iframe handling mode: relevant same-origin, all-same-origin, or none' }, + { name: 'diagnose', type: 'boolean', default: false, help: 'Print render diagnostics (frames, empty containers, XHR/API-like requests) to stderr' }, + { name: 'stdout', type: 'boolean', default: false, help: 'Print markdown to stdout instead of saving to a file' }, + ], + columns: ['title', 'author', 'publish_time', 'status', 'size', 'saved'], + func: async (page, kwargs, debug) => (await import('./browser.js')).runFetchBrowser(page, kwargs, debug), +}); + +/** + * The escalation runs the same browser session plumbing as `web fetch-browser` + * — same site, strategy and navigation policy — but returns the page content + * instead of exporting it to disk. Reusing the normalized command keeps the two + * tiers from drifting apart; it is deliberately not registered, so `webcmd list` + * still shows exactly two web commands. + */ +const escalationCommand = { + ...webFetchBrowserCommand, + func: async (page: IPage, kwargs: CommandArgs) => (await import('./browser.js')).extractPageMarkdown(page, kwargs), +} as CliCommand; + +/** + * Whether a failed HTTP fetch should be retried in a browser, and if not, why. + * + * The reason matters: `client.ts` raises these errors with a generic hint, and + * an agent that reaches one deserves to know escalation was declined rather + * than unavailable. Hosted mode is excluded on purpose — it executes adapters + * server-side, and a hosted user's machine may have no daemon or Cloak at all, + * so a local browser must never be launched behind their back. + */ +async function escalationDecision(allowBrowser: boolean, error: unknown): Promise<{ escalate: boolean; hint?: string }> { + if (!(error instanceof CliError) || !ESCALATION_CODES.has(error.code)) return { escalate: false }; + if (!allowBrowser) return { escalate: false, hint: 'Re-run without --browser false to render this page in a browser.' }; + const { shouldUseHostedMode } = await import('../hosted/config.js'); + if (shouldUseHostedMode()) return { escalate: false, hint: 'Hosted mode does not launch a local browser. Switch to local mode with: webcmd setup' }; + return { escalate: true }; +} + +/** Rethrows a declined escalation with the accurate reason attached. */ +function declined(error: unknown, hint?: string): never { + if (hint && error instanceof CliError) throw new CliError(error.code, error.message, hint); + throw error; +} + +async function escalateToBrowser(kwargs: CommandArgs, result: Partial, debug: boolean): Promise { + const { executeCommand } = await import('../execution.js'); + const page = await executeCommand(escalationCommand, { url: kwargs.url, wait: kwargs.wait ?? 3, frames: 'same-origin' }, debug) as { title: string; content: string }; + return { + status: result.status ?? 200, + requestedUrl: String(kwargs.url), + finalUrl: result.finalUrl ?? String(kwargs.url), + contentType: 'text/html', + tier: 'browser', + title: page.title, + extractionSource: 'browser', + truncated: false, + content: page.content, + }; +} export const webFetchCommand = cli({ site: 'web', name: 'fetch', access: 'read', strategy: Strategy.PUBLIC, browser: false, - description: 'Fetch a URL locally without launching a browser', defaultFormat: 'md', + description: 'Fetch a URL, escalating to a real browser only if plain HTTP is blocked', defaultFormat: 'md', args: [ - { name: 'url', type: 'string', required: true }, - { name: 'timeout', type: 'int', default: 30 }, - { name: 'max-chars', type: 'int', default: 50000 }, - { name: 'allow-private', type: 'boolean', default: false }, + { name: 'url', type: 'string', required: true, help: 'Any http or https URL' }, + { name: 'timeout', type: 'int', default: 30, help: 'Total budget in seconds across every tier' }, + { name: 'max-chars', type: 'int', default: 50000, help: 'Truncate content at this many characters (0 disables)' }, + { name: 'allow-private', type: 'boolean', default: false, help: 'Allow fetching private/loopback addresses' }, + { name: 'browser', type: 'boolean', default: true, help: 'Escalate to a real browser when the site blocks plain HTTP (--browser false to stop at HTTP)' }, + { name: 'wait', type: 'int', default: 3, help: 'Seconds to wait after page load when escalating to the browser' }, ], - func: async kwargs => webFetch({ url: String(kwargs.url), timeoutSeconds: Number(kwargs.timeout ?? 30), maxChars: Number(kwargs['max-chars'] ?? 50000), allowPrivate: kwargs['allow-private'] === true }), + func: async (kwargs, debug = false) => { + try { + return await (await loadWebFetch())(clientOptionsFromKwargs(kwargs)); + } catch (error) { + const decision = await escalationDecision(kwargs.browser !== false, error); + if (!decision.escalate) declined(error, decision.hint); + return escalateToBrowser(kwargs, {}, debug); + } + }, }); +function clientOptionsFromKwargs(kwargs: CommandArgs): WebFetchOptions { + return { + url: String(kwargs.url), + timeoutSeconds: Number(kwargs.timeout ?? 30), + maxChars: Number(kwargs['max-chars'] ?? 50000), + allowPrivate: kwargs['allow-private'] === true, + }; +} + export function formatWebFetchMarkdown(result: WebFetchResult): string { return [`# ${result.title || 'Fetched content'}`, '', `Source: ${result.requestedUrl}`, `Final URL: ${result.finalUrl}`, `Content type: ${result.contentType || 'unknown'}`, `Extraction: ${result.extractionSource}`, '', result.content].join('\n'); } -function clientOptions(argv: readonly string[]): WebFetchOptions { +function clientOptions(argv: readonly string[]): WebFetchOptions & { browser: boolean; wait: number } { const values: Record = {}; for (let index = 2; index < argv.length; index++) { const arg = argv[index]!; @@ -28,10 +134,61 @@ function clientOptions(argv: readonly string[]): WebFetchOptions { } if (typeof values.url !== 'string' || !/^https?:\/\//i.test(values.url)) throw new ArgumentError('--url must be an http or https URL'); const int = (name: string, fallback: number) => { const value = values[name]; const number = value === undefined ? fallback : Number(value); if (!Number.isInteger(number) || number < 0) throw new ArgumentError(`--${name} must be a non-negative integer`); return number; }; - return { url: values.url, timeoutSeconds: int('timeout', 30), maxChars: int('max-chars', 50000), allowPrivate: values['allow-private'] === true || values['allow-private'] === 'true' }; + return { + url: values.url, + timeoutSeconds: int('timeout', 30), + maxChars: int('max-chars', 50000), + allowPrivate: values['allow-private'] === true || values['allow-private'] === 'true', + browser: !(values.browser === 'false' || values['no-browser'] === true), + wait: int('wait', 3), + }; } +/** + * Client-owned fast path used by `src/main.ts`, which reaches this before + * adapter discovery so a plain fetch never pays the startup tax. It shares + * `webFetch` and the same escalation rule as the registered command; only the + * argv parsing and markdown rendering are its own. + */ export async function runClientOwnedWebFetch(argv: readonly string[], dependencies: { webFetch?: typeof webFetch; stdout?: NodeJS.WritableStream } = {}): Promise { - const result = await (dependencies.webFetch ?? webFetch)(clientOptions(argv)); + const options = clientOptions(argv); + const fetcher = dependencies.webFetch ?? await loadWebFetch(); + let result: WebFetchResult; + try { + result = await fetcher(options); + } catch (error) { + const decision = await escalationDecision(options.browser, error); + if (!decision.escalate) declined(error, decision.hint); + result = await escalateToBrowser({ url: options.url, wait: options.wait }, {}, false); + } (dependencies.stdout ?? process.stdout).write(`${formatWebFetchMarkdown(result)}\n`); } + +/** + * Flags the hand-rolled fast-path parser does not implement. Seeing any of them + * means falling through to the full CLI, where the registered command renders + * real help and honours `-f` — the fast path is an optimisation, never the only + * way to reach the command (#252). + */ +const FULL_CLI_FLAGS = new Set(['-h', '--help', '-f', '--format', '--trace', '-v', '--verbose']); + +export function canUseClientOwnedFastPath(argv: readonly string[]): boolean { + return argv[0] === 'web' && argv[1] === 'fetch' && !argv.some(arg => FULL_CLI_FLAGS.has(arg)); +} + +/** + * `main.ts` entry point for the fast path. Renders the same error envelope every + * other command produces instead of letting a `CliError` reach Node's default + * handler as a raw stack trace (#246), and returns the process exit code. + */ +export async function runClientOwnedWebFetchCli(argv: readonly string[]): Promise { + const { EXIT_CODES, toEnvelope } = await import('../errors.js'); + try { + await runClientOwnedWebFetch(argv); + return EXIT_CODES.SUCCESS; + } catch (error) { + const { formatErrorEnvelope } = await import('../output.js'); + process.stderr.write(formatErrorEnvelope(toEnvelope(error), { cmdName: 'web/fetch' })); + return error instanceof CliError ? error.exitCode : EXIT_CODES.GENERIC_ERROR; + } +} diff --git a/src/fetch/extract.ts b/src/fetch/extract.ts index 83e9b3d2..ef75c307 100644 --- a/src/fetch/extract.ts +++ b/src/fetch/extract.ts @@ -35,7 +35,7 @@ export function extractFetchedContent(input: ExtractFetchedContentInput): Extrac throw new CliError( 'FETCH_UNSUPPORTED_CONTENT_TYPE', `Unsupported content type: ${contentType || 'unknown'}`, - 'Use webcmd web fetch-browser for content that requires browser rendering.', + 'Use webcmd web fetch-browser to export this page from a real browser.', ); } diff --git a/src/main.ts b/src/main.ts index b496b008..1f0fa8a1 100644 --- a/src/main.ts +++ b/src/main.ts @@ -36,6 +36,10 @@ const USER_PLUGINS = path.join(os.homedir(), CONFIG_DIR_NAME, 'plugins'); // These are high-frequency or trivial paths that must not pay the startup tax. const argv = process.argv.slice(2); +// Kept in sync with FULL_CLI_FLAGS in ./fetch/command.ts. Inlined so deciding +// which path to take costs no import. +const WEB_FETCH_FULL_CLI_FLAGS = new Set(['-h', '--help', '-f', '--format', '--trace', '-v', '--verbose']); + if (typeof (globalThis as { Bun?: unknown }).Bun === 'undefined' && !isSupportedNodeVersion(process.version)) { process.stderr.write( [ @@ -77,9 +81,12 @@ if (!fastPathHandled) { } else if (argv[0] === 'skills' || argv[0] === 'update') { const { createProgram } = await import('./cli.js'); await createProgram(BUILTIN_CLIS, USER_CLIS).parseAsync(argv, { from: 'user' }); - } else if (argv[0] === 'web' && argv[1] === 'fetch') { - const { runClientOwnedWebFetch } = await import('./fetch/command.js'); - await runClientOwnedWebFetch(argv); + } else if (argv[0] === 'web' && argv[1] === 'fetch' && !argv.some(arg => WEB_FETCH_FULL_CLI_FLAGS.has(arg))) { + // `web fetch` is client-owned in both modes, so a plain fetch never pays + // discovery startup. Help, `-f` and tracing fall through to the registered + // command in ./fetch/command.ts, which owns the same flags (#252). + const { runClientOwnedWebFetchCli } = await import('./fetch/command.js'); + process.exitCode = await runClientOwnedWebFetchCli(argv); } else { const { shouldUseHostedMode } = await import('./hosted/config.js'); if (shouldUseHostedMode()) { diff --git a/src/package-exports.test.ts b/src/package-exports.test.ts index 86ffaf89..0f010299 100644 --- a/src/package-exports.test.ts +++ b/src/package-exports.test.ts @@ -1,69 +1,37 @@ /** * Regression tests for package exports. * - * Ensures adapter files use @agentrhq/webcmd public package imports - * (not fragile relative paths) and that all declared exports resolve - * to real files. + * Ensures no adapter tree sneaks back into the published package and that all + * declared exports resolve to real files. */ import { describe, it, expect } from 'vitest'; import * as fs from 'node:fs'; import * as path from 'node:path'; -import { builtinModules } from 'node:module'; import { fileURLToPath } from 'node:url'; -import ts from 'typescript'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(__dirname, '..'); -const CLIS_DIR = path.join(ROOT, 'clis', 'web'); +const CLIS_DIR = path.join(ROOT, 'clis'); const pkgJson = JSON.parse(fs.readFileSync(path.join(ROOT, 'package.json'), 'utf-8')); /** Recursively collect all JS adapter files in a directory. */ -function collectAdapterFiles(dir: string, opts?: { excludeTests?: boolean }): string[] { +function collectAdapterFiles(dir: string): string[] { const results: string[] = []; if (!fs.existsSync(dir)) return results; for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { const full = path.join(dir, entry.name); - if (entry.isDirectory()) { - results.push(...collectAdapterFiles(full, opts)); - } else if (entry.name.endsWith('.js') && !entry.name.endsWith('.d.js')) { - if (opts?.excludeTests && (entry.name.endsWith('.test.js') || entry.name.startsWith('test-'))) continue; - results.push(full); - } + if (entry.isDirectory()) results.push(...collectAdapterFiles(full)); + else if (entry.name.endsWith('.js') && !entry.name.endsWith('.d.js')) results.push(full); } return results; } -const ALLOWED_BARE_IMPORTS = new Set([ - '@agentrhq/webcmd', - ...builtinModules.flatMap((name) => name.startsWith('node:') - ? [name, name.slice(5)] - : [name, `node:${name}`]), -]); - -function isAllowedImport(specifier: string): boolean { - return specifier.startsWith('./') - || specifier.startsWith('../') - || specifier.startsWith('/') - || specifier.startsWith('@agentrhq/webcmd/') - || ALLOWED_BARE_IMPORTS.has(specifier); -} - -/** Forbidden relative import patterns that should have been replaced. - * Uses (?:\.\./)+ to catch any depth of ../ traversal. - * Covers: import/export from, vi.mock(), vi.importActual(). */ -const FORBIDDEN_PATTERNS = [ - /(?:from|mock|importActual)\s*\(?['"](?:\.\.\/)+src\//, - /(?:from|mock|importActual)\s*\(?['"](?:\.\.\/)+browser\//, - /(?:from|mock|importActual)\s*\(?['"](?:\.\.\/)+download\//, - /(?:from|mock|importActual)\s*\(?['"](?:\.\.\/)+pipeline\//, -]; - -describe('bundled web adapter imports use package exports', () => { - const adapterFiles = collectAdapterFiles(CLIS_DIR); - const runtimeAdapterFiles = collectAdapterFiles(CLIS_DIR, { excludeTests: true }); - - it('bundles the web adapter under clis/web', () => { - expect(adapterFiles.length).toBeGreaterThan(0); +describe('adapter packaging', () => { + // The `web` site used to live in clis/web. It is core TypeScript under + // src/fetch now, so import hygiene is enforced by tsc rather than by scanning + // adapter sources — but the packaging boundary below still needs asserting. + it('ships no bundled adapter tree', () => { + expect(collectAdapterFiles(CLIS_DIR)).toEqual([]); }); it('excludes adapters from package files and the install lifecycle', () => { @@ -72,43 +40,6 @@ describe('bundled web adapter imports use package exports', () => { expect(files.some(file => /^(?:clis|plugins)(?:\/|$)/.test(file))).toBe(false); expect(pkgJson.scripts.postinstall).not.toMatch(/fetch-adapters/); }); - - it('no adapter uses relative imports to src/, browser/, download/, or pipeline/', () => { - const violations: string[] = []; - for (const file of adapterFiles) { - const content = fs.readFileSync(file, 'utf-8'); - for (const pattern of FORBIDDEN_PATTERNS) { - if (pattern.test(content)) { - const rel = path.relative(ROOT, file); - const match = content.match(pattern)?.[0]; - violations.push(`${rel}: ${match}`); - } - } - } - expect(violations).toEqual([]); - }); - - it('non-test adapters only import node builtins, relative modules, or webcmd public APIs', () => { - const violations: Array<{ file: string; specifier: string }> = []; - - for (const file of runtimeAdapterFiles) { - const source = fs.readFileSync(file, 'utf-8'); - const module = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true, ts.ScriptKind.JS); - - for (const stmt of module.statements) { - if (!ts.isImportDeclaration(stmt) && !ts.isExportDeclaration(stmt)) continue; - const specifier = stmt.moduleSpecifier?.getText(module).slice(1, -1); - if (specifier && !isAllowedImport(specifier)) { - violations.push({ - file: path.relative(ROOT, file), - specifier, - }); - } - } - } - - expect(violations).toEqual([]); - }); }); describe('package.json exports resolve to real files', () => { diff --git a/src/skills.test.ts b/src/skills.test.ts index 37499ce8..a56b9941 100644 --- a/src/skills.test.ts +++ b/src/skills.test.ts @@ -50,19 +50,22 @@ function bundledSkill(name: string): string { } describe('webcmd skills content', () => { - it('keeps smart search on live discovery and explicit fetch escalation', () => { + it('keeps smart search on live discovery and automatic fetch escalation', () => { const skill = bundledSkill('smart-search'); expect(skill).toContain('webcmd list --tag search -f json'); expect(skill).toContain('webcmd plugin search'); expect(skill).toContain('webcmd plugin install'); + expect(skill).toContain('webcmd web fetch --url'); expect(skill).toContain('FETCH_BLOCKED'); - expect(skill).toContain('FETCH_REQUIRES_BROWSER'); - expect(skill).toContain('webcmd web fetch-browser'); + // `web fetch` escalates to the browser itself, so the skill must not send + // agents chasing a separate command after a block (#247). + expect(skill).not.toContain('webcmd web fetch-browser'); + expect(skill).toContain('--browser false'); expect(skill).toContain('Search Summary'); expect(skill).toMatch(/at most three.*plugin/i); expect(skill).toMatch(/up to five.*candidate/i); expect(skill).toMatch(/three.*URL.*default/i); - expect(skill).toMatch(/two.*browser fetch/i); + expect(skill).toMatch(/two.*fetch/i); expect(skill).toContain('## Site-named fast path'); expect(skill).toMatch(/cost order is mandatory when the request does not name a site/i); expect(skill).not.toContain('references/sources-'); From 9a7f8172bc4f39e4f0a52e38275a02e2f8409155 Mon Sep 17 00:00:00 2001 From: Ankit Ranjan Date: Wed, 12 Aug 2026 17:43:55 +0530 Subject: [PATCH 02/24] fix(fetch): keep web fetch output and hosted routing consistent across paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-ups found while verifying the move end to end: - `web fetch -f md` rendered a nine-column table of the result object while the fast path printed a document — the same command with two shapes. Adds a `renderMarkdown` hook on CliCommand so a command whose payload is prose can own its markdown, and points `web fetch` at the formatter the fast path already used. Both paths now emit byte-identical output. - Hosted mode kept falling through to the hosted runner once flag handling moved to the registered command, which would cloud-route a command the service cannot execute yet. Hosted mode now always takes the fast path, so its behavior is unchanged from before this refactor. - `--format=json` (equals form) slipped past the fast-path flag guard and was silently ignored; the guard now splits on `=`. - Drops the dead duplicate of the flag guard from fetch/command.ts. - The article-download e2e still invoked `web read`, renamed to `web fetch-browser` back in 0.5.x. It swallowed the resulting CLI failure and passed vacuously, so it had been testing nothing since. All six real sites now exercise the pipeline for real. Co-Authored-By: Claude Opus 5 --- src/commanderAdapter.ts | 1 + src/fetch/command.ts | 19 +++++++------------ src/main.ts | 19 ++++++++++++++++--- src/output.ts | 4 ++++ src/registry.ts | 8 ++++++++ tests/e2e/article-download-pipeline.test.ts | 6 +++--- 6 files changed, 39 insertions(+), 18 deletions(-) diff --git a/src/commanderAdapter.ts b/src/commanderAdapter.ts index 6ae9a433..586f743f 100644 --- a/src/commanderAdapter.ts +++ b/src/commanderAdapter.ts @@ -130,6 +130,7 @@ export function registerCommandToProgram( elapsed: (now() - startTime) / 1000, source: fullName(resolved), footerExtra: resolved.footerExtra?.(kwargs), + ...(resolved.renderMarkdown ? { markdown: resolved.renderMarkdown } : {}), ...(runtime.stdout ? { stdout: runtime.stdout } : {}), }); } catch (err) { diff --git a/src/fetch/command.ts b/src/fetch/command.ts index d39c4b22..f905f7cc 100644 --- a/src/fetch/command.ts +++ b/src/fetch/command.ts @@ -92,6 +92,9 @@ async function escalateToBrowser(kwargs: CommandArgs, result: Partial (isWebFetchResult(data) ? formatWebFetchMarkdown(data) : undefined), args: [ { name: 'url', type: 'string', required: true, help: 'Any http or https URL' }, { name: 'timeout', type: 'int', default: 30, help: 'Total budget in seconds across every tier' }, @@ -120,6 +123,10 @@ function clientOptionsFromKwargs(kwargs: CommandArgs): WebFetchOptions { }; } +function isWebFetchResult(data: unknown): data is WebFetchResult { + return typeof data === 'object' && data !== null && 'requestedUrl' in data && 'content' in data; +} + export function formatWebFetchMarkdown(result: WebFetchResult): string { return [`# ${result.title || 'Fetched content'}`, '', `Source: ${result.requestedUrl}`, `Final URL: ${result.finalUrl}`, `Content type: ${result.contentType || 'unknown'}`, `Extraction: ${result.extractionSource}`, '', result.content].join('\n'); } @@ -164,18 +171,6 @@ export async function runClientOwnedWebFetch(argv: readonly string[], dependenci (dependencies.stdout ?? process.stdout).write(`${formatWebFetchMarkdown(result)}\n`); } -/** - * Flags the hand-rolled fast-path parser does not implement. Seeing any of them - * means falling through to the full CLI, where the registered command renders - * real help and honours `-f` — the fast path is an optimisation, never the only - * way to reach the command (#252). - */ -const FULL_CLI_FLAGS = new Set(['-h', '--help', '-f', '--format', '--trace', '-v', '--verbose']); - -export function canUseClientOwnedFastPath(argv: readonly string[]): boolean { - return argv[0] === 'web' && argv[1] === 'fetch' && !argv.some(arg => FULL_CLI_FLAGS.has(arg)); -} - /** * `main.ts` entry point for the fast path. Renders the same error envelope every * other command produces instead of letting a `CliError` reach Node's default diff --git a/src/main.ts b/src/main.ts index 1f0fa8a1..9ca7d662 100644 --- a/src/main.ts +++ b/src/main.ts @@ -36,10 +36,23 @@ const USER_PLUGINS = path.join(os.homedir(), CONFIG_DIR_NAME, 'plugins'); // These are high-frequency or trivial paths that must not pay the startup tax. const argv = process.argv.slice(2); -// Kept in sync with FULL_CLI_FLAGS in ./fetch/command.ts. Inlined so deciding -// which path to take costs no import. +// Flags the fast path's hand-rolled parser does not implement: seeing one means +// handing `web fetch` to the registered command, which renders real help and +// honours `-f`. Inlined so the common case costs no import. `--format=json` is +// split so the equals form is caught too. const WEB_FETCH_FULL_CLI_FLAGS = new Set(['-h', '--help', '-f', '--format', '--trace', '-v', '--verbose']); +/** + * Hosted mode always keeps the fast path. Falling through there would hand the + * hosted runner a command the cloud cannot execute yet, so hosted `web fetch` + * behaves exactly as it did before this became a registered command. + */ +async function webFetchWantsFullCli(args: readonly string[]): Promise { + if (!args.some(arg => WEB_FETCH_FULL_CLI_FLAGS.has(arg.split('=')[0]!))) return false; + const { shouldUseHostedMode } = await import('./hosted/config.js'); + return !shouldUseHostedMode(); +} + if (typeof (globalThis as { Bun?: unknown }).Bun === 'undefined' && !isSupportedNodeVersion(process.version)) { process.stderr.write( [ @@ -81,7 +94,7 @@ if (!fastPathHandled) { } else if (argv[0] === 'skills' || argv[0] === 'update') { const { createProgram } = await import('./cli.js'); await createProgram(BUILTIN_CLIS, USER_CLIS).parseAsync(argv, { from: 'user' }); - } else if (argv[0] === 'web' && argv[1] === 'fetch' && !argv.some(arg => WEB_FETCH_FULL_CLI_FLAGS.has(arg))) { + } else if (argv[0] === 'web' && argv[1] === 'fetch' && !await webFetchWantsFullCli(argv)) { // `web fetch` is client-owned in both modes, so a plain fetch never pays // discovery startup. Help, `-f` and tracing fall through to the registered // command in ./fetch/command.ts, which owns the same flags (#252). diff --git a/src/output.ts b/src/output.ts index bd2f2776..471c1b6d 100644 --- a/src/output.ts +++ b/src/output.ts @@ -18,6 +18,8 @@ export interface RenderOptions { elapsed?: number; source?: string; footerExtra?: string; + /** Command-supplied markdown renderer — see CliCommand.renderMarkdown. */ + markdown?: (data: unknown) => string | undefined; } export interface ErrorRenderOptions { @@ -146,6 +148,8 @@ function formatPlain(data: unknown): string { } function formatMarkdown(data: unknown, opts: RenderOptions): string { + const custom = opts.markdown?.(data); + if (custom !== undefined) return custom.endsWith('\n') ? custom : `${custom}\n`; const rows = normalizeRows(data); if (!rows.length) return ''; if (rows.length === 1) { diff --git a/src/registry.ts b/src/registry.ts index 07368968..875a1f1b 100644 --- a/src/registry.ts +++ b/src/registry.ts @@ -68,6 +68,13 @@ interface BaseCliCommand { /** Origin of this command: 'yaml', 'ts', or plugin name. */ source?: string; footerExtra?: (kwargs: CommandArgs) => string | undefined; + /** + * Render this command's result as a markdown document instead of the default + * key/value table. For commands whose payload *is* prose — a fetched page, an + * article — the table is unreadable and differs from what the command prints + * on any other path. + */ + renderMarkdown?: (data: unknown) => string | undefined; validateArgs?: (kwargs: CommandArgs) => void; /** * Control pre-navigation and browser-session requirement. @@ -172,6 +179,7 @@ export function cli(opts: CliOptions): CliCommand { func: opts.func, pipeline: opts.pipeline, footerExtra: opts.footerExtra, + renderMarkdown: opts.renderMarkdown, validateArgs: opts.validateArgs, navigateBefore: opts.navigateBefore, siteSession: opts.siteSession, diff --git a/tests/e2e/article-download-pipeline.test.ts b/tests/e2e/article-download-pipeline.test.ts index d59cd234..76a2e7c3 100644 --- a/tests/e2e/article-download-pipeline.test.ts +++ b/tests/e2e/article-download-pipeline.test.ts @@ -1,7 +1,7 @@ /** * E2E regression tests for the HTML → Markdown article pipeline. * - * Drives real pages through `webcmd web read` and asserts the hardened + * Drives real pages through `webcmd web fetch-browser` and asserts the hardened * converter's invariants hold on the produced file: * - no base64 `data:image/…` leaks * - no ')).toBe(true)); // A CSP allow-list names third parties this page may load; it says nothing diff --git a/src/fetch/classify.ts b/src/fetch/classify.ts index ec7d08cf..fa336a6f 100644 --- a/src/fetch/classify.ts +++ b/src/fetch/classify.ts @@ -11,7 +11,8 @@ const CHALLENGE_HEADERS = /^(?:server|cf-mitigated|cf-chl-[\w-]+|x-datadome[\w-] * decide on their own at any status — including the managed-challenge * interstitial Cloudflare serves with a 200. */ -const DECISIVE_MARKERS = /cf-chl|cf-mitigated|__cf_bm|datadome|perimeterx|px-captcha|just a moment|verify you are human|checking your browser|enable javascript and cookies/i; +const DECISIVE_HEADERS = /^(?:cf-mitigated|cf-chl-[\w-]+)$/i; +const DECISIVE_BODY_MARKERS = /cf-chl|cf-mitigated|just a moment|verify you are human|checking your browser|enable javascript and cookies/i; /** * Markers that appear constantly on healthy pages: a CDN name in `server:`, @@ -19,18 +20,21 @@ const DECISIVE_MARKERS = /cf-chl|cf-mitigated|__cf_bm|datadome|perimeterx|px-cap * corroborate a status that already looks like a block, never decide alone — * that is the difference between a real block and the false positive in #283. */ -const CORROBORATING_MARKERS = /cloudflare|akamai|captcha/i; +const CORROBORATING_MARKERS = /cloudflare|datadome|perimeterx|px-captcha|akamai|captcha|__cf_bm/i; const BLOCKED_STATUSES = new Set([403, 429, 503]); export function isChallengeResponse(status: number, headers: Record, body: string): boolean { + const headerEvidence = Object.entries(headers) + .filter(([key]) => CHALLENGE_HEADERS.test(key)) + .map(([key, value]) => `${key}:${value}`) + .join('\n'); + const bodyEvidence = body.slice(0, 20_000); + if (Object.entries(headers).some(([key, value]) => DECISIVE_HEADERS.test(key) && /challenge|1/i.test(value)) || DECISIVE_BODY_MARKERS.test(bodyEvidence)) return true; const evidence = [ - ...Object.entries(headers) - .filter(([key]) => CHALLENGE_HEADERS.test(key)) - .map(([key, value]) => `${key}:${value}`), - body.slice(0, 20_000), + headerEvidence, + bodyEvidence, ].join('\n'); - if (DECISIVE_MARKERS.test(evidence)) return true; return BLOCKED_STATUSES.has(status) && CORROBORATING_MARKERS.test(evidence); } diff --git a/src/fetch/client.test.ts b/src/fetch/client.test.ts index ad6844f4..436dc797 100644 --- a/src/fetch/client.test.ts +++ b/src/fetch/client.test.ts @@ -1,12 +1,14 @@ import { describe, expect, it, vi } from 'vitest'; +import { CliError } from '../errors.js'; import { webFetch } from './client.js'; function response(body: string, status = 200, headers: Record = { 'content-type': 'text/plain' }) { return new Response(body, { status, headers }); } +const safeProxy = (close = vi.fn().mockResolvedValue(undefined), policyError: () => Error | undefined = () => undefined) => ({ url: 'http://proxy', close, policyError }); describe('webFetch', () => { it('uses healthy plain responses without escalation', async () => { const plainFetch = vi.fn().mockResolvedValue(response('ok')); const createImpit = vi.fn(); - const result = await webFetch({ url: 'https://example.com', timeoutSeconds: 5, maxChars: 0, allowPrivate: false }, { plainFetch, createImpit, createSafeProxy: async () => ({ url: 'http://proxy', close: async () => {} }) }); + const result = await webFetch({ url: 'https://example.com', timeoutSeconds: 5, maxChars: 0, allowPrivate: false }, { plainFetch, createImpit, createSafeProxy: async () => safeProxy() }); expect(result).toMatchObject({ tier: 'plain', content: 'ok' }); expect(createImpit).not.toHaveBeenCalled(); }); @@ -14,7 +16,7 @@ describe('webFetch', () => { const first = { fetch: vi.fn().mockResolvedValue(response('challenge', 403, { server: 'cloudflare', 'content-type': 'text/plain' })) }; const second = { fetch: vi.fn().mockResolvedValue(response('ok')) }; const createImpit = vi.fn().mockReturnValueOnce(first).mockReturnValueOnce(second); - const result = await webFetch({ url: 'https://example.com', timeoutSeconds: 5, maxChars: 0, allowPrivate: false }, { plainFetch: vi.fn().mockResolvedValue(response('challenge', 403, { server: 'cloudflare', 'content-type': 'text/plain' })), createImpit, createSafeProxy: async () => ({ url: 'http://proxy', close: async () => {} }) }); + const result = await webFetch({ url: 'https://example.com', timeoutSeconds: 5, maxChars: 0, allowPrivate: false }, { plainFetch: vi.fn().mockResolvedValue(response('challenge', 403, { server: 'cloudflare', 'content-type': 'text/plain' })), createImpit, createSafeProxy: async () => safeProxy() }); expect(result).toMatchObject({ tier: 'impit', profile: 'firefox', content: 'ok' }); expect(createImpit).toHaveBeenNthCalledWith(1, expect.objectContaining({ browser: 'chrome' })); expect(createImpit).toHaveBeenNthCalledWith(2, expect.objectContaining({ browser: 'firefox' })); @@ -23,8 +25,8 @@ describe('webFetch', () => { const close = vi.fn().mockResolvedValue(undefined); await expect(webFetch({ url: 'https://example.com', timeoutSeconds: 5, maxChars: 0, allowPrivate: false }, { plainFetch: vi.fn().mockRejectedValue(new Error('boom')), - createImpit: vi.fn(), - createSafeProxy: async () => ({ url: 'http://proxy', close }), + createImpit: vi.fn(() => ({ fetch: vi.fn().mockRejectedValue(new Error('boom')) })), + createSafeProxy: async () => safeProxy(close), })).rejects.toThrow('boom'); expect(close).toHaveBeenCalledOnce(); }); @@ -32,8 +34,8 @@ describe('webFetch', () => { const abort = Object.assign(new Error('The operation was aborted'), { name: 'TimeoutError' }); await expect(webFetch({ url: 'https://example.com', timeoutSeconds: 5, maxChars: 0, allowPrivate: false }, { plainFetch: vi.fn().mockRejectedValue(abort), - createImpit: vi.fn(), - createSafeProxy: async () => ({ url: 'http://proxy', close: async () => {} }), + createImpit: vi.fn(() => ({ fetch: vi.fn().mockRejectedValue(Object.assign(new Error('connect ECONNREFUSED'), { code: 'ECONNREFUSED' })) })), + createSafeProxy: async () => safeProxy(), })).rejects.toMatchObject({ code: 'TIMEOUT', message: 'web fetch timed out after 5s' }); }); it('reports an impit-shaped deadline as a structured timeout', async () => { @@ -42,15 +44,72 @@ describe('webFetch', () => { const impitTimeout = new Error('error sending request for url (https://example.com/): operation timed out'); await expect(webFetch({ url: 'https://example.com', timeoutSeconds: 0.05, maxChars: 0, allowPrivate: false }, { plainFetch: vi.fn().mockImplementation(async () => { await new Promise(done => setTimeout(done, 80)); throw impitTimeout; }), - createImpit: vi.fn(), - createSafeProxy: async () => ({ url: 'http://proxy', close: async () => {} }), + createImpit: vi.fn(() => ({ fetch: vi.fn().mockRejectedValue(Object.assign(new Error('connect ECONNREFUSED'), { code: 'ECONNREFUSED' })) })), + createSafeProxy: async () => safeProxy(), })).rejects.toMatchObject({ code: 'TIMEOUT', message: 'web fetch timed out after 0.05s' }); }); it('does not relabel a failure that happened with budget left', async () => { await expect(webFetch({ url: 'https://example.com', timeoutSeconds: 30, maxChars: 0, allowPrivate: false }, { plainFetch: vi.fn().mockRejectedValue(Object.assign(new Error('connect ECONNREFUSED'), { code: 'ECONNREFUSED' })), - createImpit: vi.fn(), - createSafeProxy: async () => ({ url: 'http://proxy', close: async () => {} }), + createImpit: vi.fn(() => ({ fetch: vi.fn().mockRejectedValue(Object.assign(new Error('connect ECONNREFUSED'), { code: 'ECONNREFUSED' })) })), + createSafeProxy: async () => safeProxy(), })).rejects.toThrow('connect ECONNREFUSED'); }); }); + +describe('webFetch fixed non-browser ladder', () => { + const options = { url: 'https://example.com', timeoutSeconds: 5, maxChars: 0, allowPrivate: false }; + const challenge = () => response('Just a moment', 403, { server: 'cloudflare', 'content-type': 'text/plain' }); + + it.each([ + ['plain challenge reaches Chrome', [challenge()], [response('chrome')], ['chrome'], { tier: 'impit', profile: 'chrome' }], + ['two challenges reach Firefox', [challenge()], [challenge(), response('firefox')], ['chrome', 'firefox'], { tier: 'impit', profile: 'firefox' }], + ])('%s', async (_name, plainResponses, impitResponses, createdProfiles, expected) => { + const impits = impitResponses.map(value => ({ fetch: vi.fn().mockResolvedValue(value) })); + const createImpit = vi.fn().mockImplementation(() => impits.shift()); + const result = await webFetch(options, { plainFetch: vi.fn().mockResolvedValue(plainResponses[0]), createImpit, createSafeProxy: async () => safeProxy() }); + expect(createImpit.mock.calls.map(([value]) => value.browser)).toEqual(createdProfiles); + expect(result).toMatchObject(expected); + }); + + it('advances transport failures through Firefox', async () => { + const createdProfiles: string[] = []; + const createImpit = vi.fn(({ browser }) => { + createdProfiles.push(browser); + return { fetch: vi.fn().mockImplementation(() => browser === 'chrome' ? Promise.reject(new Error('TLS')) : Promise.resolve(response('firefox'))) }; + }); + const result = await webFetch(options, { plainFetch: vi.fn().mockRejectedValue(new Error('socket')), createImpit, createSafeProxy: async () => safeProxy() }); + expect(createdProfiles).toEqual(['chrome', 'firefox']); + expect(result).toMatchObject({ tier: 'impit', profile: 'firefox' }); + }); + + it('gives Chrome and Firefox decreasing positive timeouts from one deadline', async () => { + const createImpit = vi.fn((_options: { browser: 'chrome' | 'firefox'; proxyUrl: string; timeout: number }) => ({ fetch: vi.fn().mockImplementation(async () => { await new Promise(done => setTimeout(done, 5)); return challenge(); }) })); + await expect(webFetch({ ...options, timeoutSeconds: 1 }, { plainFetch: vi.fn().mockResolvedValue(challenge()), createImpit, createSafeProxy: async () => safeProxy() })).rejects.toMatchObject({ code: 'FETCH_BLOCKED' }); + const timeouts = createImpit.mock.calls.map(([value]) => value.timeout); + expect(timeouts).toHaveLength(2); + expect(timeouts[0]).toBeGreaterThan(timeouts[1]); + expect(timeouts[1]).toBeGreaterThan(0); + }); + + it('stops terminal errors and uses the explicit Session workflow', async () => { + const createImpit = vi.fn(); + await expect(webFetch(options, { plainFetch: vi.fn().mockResolvedValue(response('
      ')), createImpit, createSafeProxy: async () => safeProxy() })).rejects.toMatchObject({ code: 'FETCH_REQUIRES_BROWSER', hint: expect.stringContaining('session create') }); + expect(createImpit).not.toHaveBeenCalled(); + await expect(webFetch(options, { plainFetch: vi.fn().mockRejectedValue(new CliError('FETCH_BODY_TOO_LARGE', 'large')), createImpit, createSafeProxy: async () => safeProxy() })).rejects.toMatchObject({ code: 'FETCH_BODY_TOO_LARGE' }); + expect(createImpit).not.toHaveBeenCalled(); + }); + + it('stops for a proxy policy error and closes once', async () => { + const close = vi.fn().mockResolvedValue(undefined); + const createImpit = vi.fn(); + await expect(webFetch(options, { plainFetch: vi.fn().mockRejectedValue(new Error('proxy failure')), createImpit, createSafeProxy: async () => safeProxy(close, () => new Error('Unsafe fetch destination: 127.0.0.1')) })).rejects.toMatchObject({ code: 'FETCH_UNSAFE_ADDRESS' }); + expect(createImpit).not.toHaveBeenCalled(); + expect(close).toHaveBeenCalledOnce(); + }); + + it('returns blocked after three completed challenges with the explicit Session workflow', async () => { + const createImpit = vi.fn().mockReturnValueOnce({ fetch: vi.fn().mockResolvedValue(challenge()) }).mockReturnValueOnce({ fetch: vi.fn().mockResolvedValue(challenge()) }); + await expect(webFetch(options, { plainFetch: vi.fn().mockResolvedValue(challenge()), createImpit, createSafeProxy: async () => safeProxy() })).rejects.toMatchObject({ code: 'FETCH_BLOCKED', hint: expect.stringContaining('browser run --stdin') }); + }); +}); diff --git a/src/fetch/client.ts b/src/fetch/client.ts index f42da73b..1e762864 100644 --- a/src/fetch/client.ts +++ b/src/fetch/client.ts @@ -7,8 +7,8 @@ import { isChallengeResponse, isJavaScriptShell } from './classify.js'; export interface WebFetchOptions { url: string; timeoutSeconds: number; maxChars: number; allowPrivate: boolean; } export interface WebFetchResult { - status: number; requestedUrl: string; finalUrl: string; contentType: string; tier: 'plain' | 'impit' | 'browser'; profile?: 'chrome' | 'firefox'; - title: string; extractionSource: ExtractFetchedContentResult['source'] | 'browser'; truncated: boolean; content: string; + status: number; requestedUrl: string; finalUrl: string; contentType: string; tier: 'plain' | 'impit'; profile?: 'chrome' | 'firefox'; + title: string; extractionSource: ExtractFetchedContentResult['source']; truncated: boolean; content: string; } type ResponseLike = Pick & { body?: ReadableStream | null; bytes?: () => Promise; }; type FetchLike = (url: string, options?: Record) => Promise; @@ -19,6 +19,7 @@ export interface WebFetchDependencies { createSafeProxy?: (options: { allowPrivate: boolean }) => Promise; } const MAX_BODY_BYTES = 10 * 1024 * 1024; +const BROWSER_WORKFLOW = 'Create a browser Session with `webcmd --profile work session create`, then navigate with `webcmd --profile work --session browser run --stdin`.'; function headersOf(response: ResponseLike): Record { return Object.fromEntries(response.headers.entries()); } async function readBody(response: ResponseLike): Promise { @@ -46,26 +47,35 @@ export async function webFetch(options: WebFetchOptions, dependencies: WebFetchD const plainFetch = dependencies.plainFetch ?? ((url, init) => fetch(url, init)); const createImpit = dependencies.createImpit ?? (impitOptions => new Impit(impitOptions)); try { - let response = await plainFetch(options.url, { redirect: 'manual', dispatcher: new ProxyAgent(proxy.url), signal: AbortSignal.timeout(remaining()) }); - let body = await readBody(response); - let tier: WebFetchResult['tier'] = 'plain'; let profile: WebFetchResult['profile']; - if (isJavaScriptShell(body)) throw new CliError('FETCH_REQUIRES_BROWSER', 'This page requires browser rendering.', 'The browser tier renders this page; web fetch escalates to it automatically.'); - if (isChallengeResponse(response.status, headersOf(response), body)) { - // ponytail: impit's timeout covers the request, not the body stream, so a - // trickling escalation body can outlive the budget. Race readBody against - // the deadline if that shows up in practice. - for (const browser of ['chrome', 'firefox'] as const) { - const impit = createImpit({ browser, proxyUrl: proxy.url, timeout: remaining() }); - response = await impit.fetch(options.url, { redirect: 'manual', timeout: remaining() }); - body = await readBody(response); tier = 'impit'; profile = browser; - if (isJavaScriptShell(body)) throw new CliError('FETCH_REQUIRES_BROWSER', 'This page requires browser rendering.', 'The browser tier renders this page; web fetch escalates to it automatically.'); - if (!isChallengeResponse(response.status, headersOf(response), body)) break; + const ladder = [undefined, 'chrome', 'firefox'] as const; + let lastTransport: unknown; + for (const browser of ladder) { + try { + const timeout = remaining(); + const response = browser + ? await createImpit({ browser, proxyUrl: proxy.url, timeout }).fetch(options.url, { redirect: 'manual', timeout: remaining() }) + : await plainFetch(options.url, { redirect: 'manual', dispatcher: new ProxyAgent(proxy.url), signal: AbortSignal.timeout(timeout) }); + const policyError = proxy.policyError(); + if (policyError) throw new CliError('FETCH_UNSAFE_ADDRESS', policyError.message); + const body = await readBody(response); + if (proxy.policyError()) throw new CliError('FETCH_UNSAFE_ADDRESS', proxy.policyError()!.message); + if (isJavaScriptShell(body)) throw new CliError('FETCH_REQUIRES_BROWSER', 'This page requires browser rendering.', BROWSER_WORKFLOW); + if (isChallengeResponse(response.status, headersOf(response), body)) { + if (browser === 'firefox') throw new CliError('FETCH_BLOCKED', 'The site blocked non-browser fetches.', BROWSER_WORKFLOW); + continue; + } + const extracted = extractFetchedContent({ body, contentType: response.headers.get('content-type') ?? '', url: options.url }); + const clipped = truncate(extracted.content, options.maxChars); + return { status: response.status, requestedUrl: options.url, finalUrl: response.url || options.url, contentType: response.headers.get('content-type') ?? '', tier: browser ? 'impit' : 'plain', ...(browser && { profile: browser }), title: extracted.title, extractionSource: extracted.source, truncated: clipped.truncated, content: clipped.content }; + } catch (error) { + const policyError = proxy.policyError(); + if (policyError) throw new CliError('FETCH_UNSAFE_ADDRESS', policyError.message); + const mapped = asFetchError(error, options.timeoutSeconds, deadline); + if (mapped instanceof CliError) throw mapped; + lastTransport = mapped; } - if (isChallengeResponse(response.status, headersOf(response), body)) throw new CliError('FETCH_BLOCKED', 'The site blocked non-browser fetches.', 'The browser tier renders this page; web fetch escalates to it automatically.'); } - const extracted = extractFetchedContent({ body, contentType: response.headers.get('content-type') ?? '', url: options.url }); - const clipped = truncate(extracted.content, options.maxChars); - return { status: response.status, requestedUrl: options.url, finalUrl: response.url || options.url, contentType: response.headers.get('content-type') ?? '', tier, ...(profile && { profile }), title: extracted.title, extractionSource: extracted.source, truncated: clipped.truncated, content: clipped.content }; + throw lastTransport; } catch (error) { throw asFetchError(error, options.timeoutSeconds, deadline); } finally { await proxy.close(); } diff --git a/src/fetch/extract.ts b/src/fetch/extract.ts index ef75c307..2e9976b1 100644 --- a/src/fetch/extract.ts +++ b/src/fetch/extract.ts @@ -35,7 +35,7 @@ export function extractFetchedContent(input: ExtractFetchedContentInput): Extrac throw new CliError( 'FETCH_UNSUPPORTED_CONTENT_TYPE', `Unsupported content type: ${contentType || 'unknown'}`, - 'Use webcmd web fetch-browser to export this page from a real browser.', + 'Create a browser Session with `webcmd --profile work session create`, then navigate with `webcmd --profile work --session browser run --stdin`.', ); } diff --git a/src/fetch/safe-proxy.test.ts b/src/fetch/safe-proxy.test.ts index 5b8e51db..63da1cd5 100644 --- a/src/fetch/safe-proxy.test.ts +++ b/src/fetch/safe-proxy.test.ts @@ -72,3 +72,25 @@ describe('createSafeProxy close', () => { await new Promise(done => upstream.close(() => done())); }); }); + +describe('createSafeProxy policy errors', () => { + it('records the first rejected private destination', async () => { + const proxy = await createSafeProxy({ allowPrivate: false }); + const client = net.connect({ host: '127.0.0.1', port: Number(new URL(proxy.url).port) }); + const reply = await new Promise(done => { + let data = ''; + client.on('data', chunk => { data += chunk; }); + client.on('end', () => done(data)); + client.write('CONNECT 127.0.0.1:443 HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n'); + }); + expect(reply).toContain('403 Forbidden'); + expect(proxy.policyError()?.message).toContain('Unsafe fetch destination'); + await proxy.close(); + }); + + it('leaves the policy slot empty when private addresses are allowed', async () => { + const proxy = await createSafeProxy({ allowPrivate: true }); + expect(proxy.policyError()).toBeUndefined(); + await proxy.close(); + }); +}); diff --git a/src/fetch/safe-proxy.ts b/src/fetch/safe-proxy.ts index 2d82527f..b6adb77d 100644 --- a/src/fetch/safe-proxy.ts +++ b/src/fetch/safe-proxy.ts @@ -3,7 +3,7 @@ import * as http from 'node:http'; import * as net from 'node:net'; import type { Duplex } from 'node:stream'; -export interface SafeProxy { url: string; close(): Promise; } +export interface SafeProxy { url: string; close(): Promise; policyError(): Error | undefined; } export interface SafeProxyOptions { allowPrivate?: boolean; lookup?: typeof dnsLookup; } export function isSafeAddress(address: string): boolean { @@ -49,6 +49,7 @@ export async function createSafeProxy(options: SafeProxyOptions = {}): Promise(socket: T): T => { if (closing) { socket.destroy(); return socket; } sockets.add(socket); @@ -69,7 +70,10 @@ export async function createSafeProxy(options: SafeProxyOptions = {}): Promise response.destroy(error)); request.pipe(upstream); - } catch (error) { response.writeHead(403).end(error instanceof Error ? error.message : 'Unsafe fetch destination'); } + } catch (error) { + if (error instanceof Error && error.message.startsWith('Unsafe fetch destination:')) firstPolicyError ??= error; + response.writeHead(403).end(error instanceof Error ? error.message : 'Unsafe fetch destination'); + } }); server.on('connection', track); server.on('connect', async (request, client, head) => { @@ -82,7 +86,10 @@ export async function createSafeProxy(options: SafeProxyOptions = {}): Promise { client.write('HTTP/1.1 200 Connection Established\r\n\r\n'); if (head.length) upstream.write(head); upstream.pipe(client); client.pipe(upstream); }); upstream.once('error', error => client.destroy(error)); - } catch (error) { client.end(`HTTP/1.1 403 Forbidden\r\n\r\n${error instanceof Error ? error.message : ''}`); } + } catch (error) { + if (error instanceof Error && error.message.startsWith('Unsafe fetch destination:')) firstPolicyError ??= error; + client.end(`HTTP/1.1 403 Forbidden\r\n\r\n${error instanceof Error ? error.message : ''}`); + } }); await new Promise((resolveListen, reject) => { server.once('error', reject); server.listen(0, '127.0.0.1', () => resolveListen()); }); const address = server.address(); @@ -90,6 +97,7 @@ export async function createSafeProxy(options: SafeProxyOptions = {}): Promise | undefined; return { url: `http://127.0.0.1:${address.port}`, + policyError: () => firstPolicyError, // Idempotent: a second close() awaits the first rather than asking an // already-stopped server to close again. close: () => (closed ??= new Promise((resolveClose, reject) => { From 087ea160b7abc151036501428ce92355508c9bed Mon Sep 17 00:00:00 2001 From: beubax Date: Wed, 12 Aug 2026 23:28:59 +0530 Subject: [PATCH 09/24] fix(fetch): bound body reads by deadline --- src/fetch/client.test.ts | 17 +++++++++++++++++ src/fetch/client.ts | 16 ++++++++++++---- src/fetch/extract.test.ts | 10 ++++++++++ src/fetch/extract.ts | 5 ++++- 4 files changed, 43 insertions(+), 5 deletions(-) diff --git a/src/fetch/client.test.ts b/src/fetch/client.test.ts index 436dc797..cf5d8565 100644 --- a/src/fetch/client.test.ts +++ b/src/fetch/client.test.ts @@ -48,6 +48,23 @@ describe('webFetch', () => { createSafeProxy: async () => safeProxy(), })).rejects.toMatchObject({ code: 'TIMEOUT', message: 'web fetch timed out after 0.05s' }); }); + it('times out a body read and closes the proxy', async () => { + const close = vi.fn().mockResolvedValue(undefined); + const cancel = vi.fn(); + const hanging = { + status: 200, + headers: new Headers({ 'content-type': 'text/plain' }), + url: 'https://example.com', + body: new ReadableStream({ pull: () => new Promise(() => {}), cancel }), + }; + await expect(webFetch({ url: 'https://example.com', timeoutSeconds: 0.05, maxChars: 0, allowPrivate: false }, { + plainFetch: vi.fn().mockResolvedValue(hanging), + createImpit: vi.fn(), + createSafeProxy: async () => safeProxy(close), + })).rejects.toMatchObject({ code: 'TIMEOUT' }); + expect(cancel).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + }); it('does not relabel a failure that happened with budget left', async () => { await expect(webFetch({ url: 'https://example.com', timeoutSeconds: 30, maxChars: 0, allowPrivate: false }, { plainFetch: vi.fn().mockRejectedValue(Object.assign(new Error('connect ECONNREFUSED'), { code: 'ECONNREFUSED' })), diff --git a/src/fetch/client.ts b/src/fetch/client.ts index 1e762864..92211d72 100644 --- a/src/fetch/client.ts +++ b/src/fetch/client.ts @@ -22,16 +22,24 @@ const MAX_BODY_BYTES = 10 * 1024 * 1024; const BROWSER_WORKFLOW = 'Create a browser Session with `webcmd --profile work session create`, then navigate with `webcmd --profile work --session browser run --stdin`.'; function headersOf(response: ResponseLike): Record { return Object.fromEntries(response.headers.entries()); } -async function readBody(response: ResponseLike): Promise { +function beforeDeadline(promise: Promise, deadline: number, timeoutSeconds: number, cancel?: () => void): Promise { + const ms = deadline - Date.now(); + if (ms <= 0) { cancel?.(); return Promise.reject(new TimeoutError('web fetch', timeoutSeconds)); } + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { cancel?.(); reject(new TimeoutError('web fetch', timeoutSeconds)); }, ms); + promise.then(value => { clearTimeout(timer); resolve(value); }, error => { clearTimeout(timer); reject(error); }); + }); +} +async function readBody(response: ResponseLike, deadline: number, timeoutSeconds: number): Promise { if (!response.body && response.bytes) { - const bytes = await response.bytes(); + const bytes = await beforeDeadline(response.bytes(), deadline, timeoutSeconds); if (bytes.byteLength > MAX_BODY_BYTES) throw new CliError('FETCH_BODY_TOO_LARGE', 'Fetched body exceeds 10 MiB'); return new TextDecoder().decode(bytes); } const reader = response.body?.getReader(); if (!reader) return ''; const chunks: Uint8Array[] = []; let size = 0; - while (true) { const { done, value } = await reader.read(); if (done) break; size += value.byteLength; if (size > MAX_BODY_BYTES) { await reader.cancel(); throw new CliError('FETCH_BODY_TOO_LARGE', 'Fetched body exceeds 10 MiB'); } chunks.push(value); } + while (true) { const { done, value } = await beforeDeadline(reader.read(), deadline, timeoutSeconds, () => { void reader.cancel(); }); if (done) break; size += value.byteLength; if (size > MAX_BODY_BYTES) { await reader.cancel(); throw new CliError('FETCH_BODY_TOO_LARGE', 'Fetched body exceeds 10 MiB'); } chunks.push(value); } return new TextDecoder().decode(Buffer.concat(chunks)); } function truncate(content: string, limit: number): { content: string; truncated: boolean } { @@ -57,7 +65,7 @@ export async function webFetch(options: WebFetchOptions, dependencies: WebFetchD : await plainFetch(options.url, { redirect: 'manual', dispatcher: new ProxyAgent(proxy.url), signal: AbortSignal.timeout(timeout) }); const policyError = proxy.policyError(); if (policyError) throw new CliError('FETCH_UNSAFE_ADDRESS', policyError.message); - const body = await readBody(response); + const body = await readBody(response, deadline, options.timeoutSeconds); if (proxy.policyError()) throw new CliError('FETCH_UNSAFE_ADDRESS', proxy.policyError()!.message); if (isJavaScriptShell(body)) throw new CliError('FETCH_REQUIRES_BROWSER', 'This page requires browser rendering.', BROWSER_WORKFLOW); if (isChallengeResponse(response.status, headersOf(response), body)) { diff --git a/src/fetch/extract.test.ts b/src/fetch/extract.test.ts index 945abf7d..404b9444 100644 --- a/src/fetch/extract.test.ts +++ b/src/fetch/extract.test.ts @@ -25,4 +25,14 @@ describe('extractFetchedContent', () => { } throw new Error('expected unsupported content type'); }); + + it('does not recommend browser rendering for binary downloads', () => { + try { + extractFetchedContent({ url: 'https://example.com/a', contentType: 'application/octet-stream', body: '' }); + } catch (error) { + expect(error).toMatchObject({ code: 'FETCH_UNSUPPORTED_CONTENT_TYPE', hint: undefined }); + return; + } + throw new Error('expected unsupported content type'); + }); }); diff --git a/src/fetch/extract.ts b/src/fetch/extract.ts index 2e9976b1..6ebd2c29 100644 --- a/src/fetch/extract.ts +++ b/src/fetch/extract.ts @@ -32,10 +32,13 @@ export function extractFetchedContent(input: ExtractFetchedContentInput): Extrac return { title: '', content: input.body, source: 'raw' }; } if (!['text/html', 'application/xhtml+xml', ''].includes(contentType)) { + const hint = contentType === 'application/pdf' + ? 'Create a browser Session with `webcmd --profile work session create`, then navigate with `webcmd --profile work --session browser run --stdin`.' + : undefined; throw new CliError( 'FETCH_UNSUPPORTED_CONTENT_TYPE', `Unsupported content type: ${contentType || 'unknown'}`, - 'Create a browser Session with `webcmd --profile work session create`, then navigate with `webcmd --profile work --session browser run --stdin`.', + hint, ); } From 5a10773ebba7b0d3f46cc87be3181bd40100859b Mon Sep 17 00:00:00 2001 From: beubax Date: Wed, 12 Aug 2026 23:33:40 +0530 Subject: [PATCH 10/24] feat(fetch): publish client-owned core manifest entry --- cli-manifest.json | 117 ++------------------------------ src/build-manifest.test.ts | 17 +++++ src/build-manifest.ts | 17 +++-- src/hosted/availability.test.ts | 5 ++ src/hosted/availability.ts | 4 +- src/hosted/contract.test.ts | 16 +++++ src/hosted/contract.ts | 1 + src/manifest-types.ts | 2 + src/package-exports.test.ts | 30 ++++++++ 9 files changed, 91 insertions(+), 118 deletions(-) diff --git a/cli-manifest.json b/cli-manifest.json index a3c38d77..c3a91a9b 100644 --- a/cli-manifest.json +++ b/cli-manifest.json @@ -2,7 +2,7 @@ { "site": "web", "name": "fetch", - "description": "Fetch a URL, escalating to a real browser only if plain HTTP is blocked", + "description": "Fetch a URL with local HTTP clients", "access": "read", "strategy": "public", "browser": false, @@ -11,138 +11,33 @@ "name": "url", "type": "string", "required": true, - "help": "Any http or https URL" + "help": "HTTP or HTTPS URL to fetch" }, { "name": "timeout", "type": "int", "default": 30, "required": false, - "help": "Total budget in seconds across every tier" + "help": "Total fetch budget in seconds" }, { "name": "max-chars", "type": "int", "default": 50000, "required": false, - "help": "Truncate content at this many characters (0 disables)" + "help": "Maximum extracted characters; 0 disables truncation" }, { "name": "allow-private", "type": "boolean", "default": false, "required": false, - "help": "Allow fetching private/loopback addresses" - }, - { - "name": "browser", - "type": "boolean", - "default": true, - "required": false, - "help": "Escalate to a real browser when the site blocks plain HTTP (--browser false to stop at HTTP)" - }, - { - "name": "wait", - "type": "int", - "default": 3, - "required": false, - "help": "Seconds to wait after page load when escalating to the browser" + "help": "Allow private and loopback destinations" } ], "defaultFormat": "md", "type": "js", - "packageExport": "./fetch/command" - }, - { - "site": "web", - "name": "fetch-browser", - "description": "Fetch any web page in a real browser and export as Markdown", - "access": "read", - "strategy": "cookie", - "browser": true, - "args": [ - { - "name": "url", - "type": "str", - "required": true, - "help": "Any web page URL" - }, - { - "name": "output", - "type": "str", - "default": "./web-articles", - "required": false, - "help": "Output directory" - }, - { - "name": "download-images", - "type": "boolean", - "default": true, - "required": false, - "help": "Download images locally" - }, - { - "name": "wait", - "type": "int", - "default": 3, - "required": false, - "help": "Seconds to wait after page load" - }, - { - "name": "wait-for", - "type": "str", - "required": false, - "valueRequired": true, - "help": "CSS selector to wait for in the main document or same-origin iframes" - }, - { - "name": "wait-until", - "type": "str", - "default": "domstable", - "required": false, - "help": "Readiness policy after navigation: domstable or networkidle", - "choices": [ - "domstable", - "networkidle" - ] - }, - { - "name": "frames", - "type": "str", - "default": "same-origin", - "required": false, - "help": "Iframe handling mode: relevant same-origin, all-same-origin, or none", - "choices": [ - "same-origin", - "all-same-origin", - "none" - ] - }, - { - "name": "diagnose", - "type": "boolean", - "default": false, - "required": false, - "help": "Print render diagnostics (frames, empty containers, XHR/API-like requests) to stderr" - }, - { - "name": "stdout", - "type": "boolean", - "default": false, - "required": false, - "help": "Print markdown to stdout instead of saving to a file" - } - ], - "columns": [ - "title", - "author", - "publish_time", - "status", - "size", - "saved" - ], - "type": "js", - "navigateBefore": false, + "clientOwned": true, "packageExport": "./fetch/command" } ] diff --git a/src/build-manifest.test.ts b/src/build-manifest.test.ts index 403cae84..3887c26c 100644 --- a/src/build-manifest.test.ts +++ b/src/build-manifest.test.ts @@ -6,6 +6,7 @@ import { cli, getRegistry, Strategy } from './registry.js'; import { ManifestImportError, buildManifestArtifacts, + coreCommandEntries, diffRemovedEntries, findManifestMetadataIssues, loadManifestEntries, @@ -36,6 +37,22 @@ describe('manifest helper rules', () => { expect(serializeManifest([])).toBe('[]\n'); }); + it('serializes web fetch as the single client-owned core command', async () => { + const entries = await coreCommandEntries(); + + expect(entries).toHaveLength(1); + expect(entries).toEqual([ + expect.objectContaining({ + site: 'web', + name: 'fetch', + clientOwned: true, + packageExport: './fetch/command', + }), + ]); + expect(entries[0]).not.toHaveProperty('modulePath'); + expect(entries[0]).not.toHaveProperty('sourceFile'); + }); + it('skips TS files that do not register a cli', () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'webcmd-manifest-')); tempDirs.push(dir); diff --git a/src/build-manifest.ts b/src/build-manifest.ts index c76019b6..8a20d7b3 100644 --- a/src/build-manifest.ts +++ b/src/build-manifest.ts @@ -111,7 +111,10 @@ function isCliCommandValue(value: unknown, site: string): value is CliCommand { && Array.isArray(value.args); } -function toManifestEntry(cmd: CliCommand, modulePath?: string, sourceFile?: string): ManifestEntry { +function toManifestEntry( + cmd: CliCommand, + adapterPath?: Pick, +): ManifestEntry { return { site: cmd.site, name: cmd.name, @@ -128,8 +131,8 @@ function toManifestEntry(cmd: CliCommand, modulePath?: string, sourceFile?: stri ...(cmd.keywords?.length ? { keywords: [...cmd.keywords] } : {}), defaultFormat: cmd.defaultFormat, type: 'js', - modulePath, - sourceFile, + ...adapterPath, + ...(cmd.clientOwned ? { clientOwned: true } : {}), navigateBefore: cmd.navigateBefore, siteSession: cmd.siteSession, freshPage: cmd.freshPage, @@ -195,7 +198,7 @@ export async function loadManifestEntries( return true; }) .sort((a, b) => a.name.localeCompare(b.name)) - .map(cmd => toManifestEntry(cmd, modulePath, sourceRelative)); + .map(cmd => toManifestEntry(cmd, { modulePath, sourceFile: sourceRelative })); } catch (err) { throw new ManifestImportError(filePath, err); } @@ -272,7 +275,7 @@ export async function coreCommandEntries( ): Promise { await importer(pathToFileURL(path.join(PACKAGE_ROOT, 'src/fetch/command.ts')).href); return [...getRegistry().values()] - .filter(cmd => CORE_SITE_EXPORTS.has(cmd.site)) + .filter(cmd => cmd.clientOwned && CORE_SITE_EXPORTS.has(cmd.site)) .sort((a, b) => a.site.localeCompare(b.site) || a.name.localeCompare(b.name)) .map(cmd => ({ ...toManifestEntry(cmd), packageExport: CORE_SITE_EXPORTS.get(cmd.site)! })); } @@ -280,7 +283,9 @@ export async function coreCommandEntries( export async function buildManifest(): Promise { const scanned = await scanClisDir(LEGACY_CLIS_DIR); const core = await coreCommandEntries(); - const entries = [...scanned.entries, ...core].sort( + const entriesByCommand = new Map(scanned.entries.map(entry => [`${entry.site}/${entry.name}`, entry])); + for (const entry of core) entriesByCommand.set(`${entry.site}/${entry.name}`, entry); + const entries = [...entriesByCommand.values()].sort( (a, b) => a.site.localeCompare(b.site) || a.name.localeCompare(b.name), ); return { entries, failures: scanned.failures }; diff --git a/src/hosted/availability.test.ts b/src/hosted/availability.test.ts index df4ce587..e46dc9e7 100644 --- a/src/hosted/availability.test.ts +++ b/src/hosted/availability.test.ts @@ -165,6 +165,11 @@ function exceptionDiff(actual: readonly string[], expected: readonly string[]) { } describe('hosted availability', () => { + it('keeps client-owned commands local before strategy and domain classification', () => { + expect(deriveHostedAvailability({ clientOwned: true, strategy: 'public', domain: 'example.com' })) + .toEqual({ mode: 'local-only', reason: 'client-owned' }); + }); + it('derives decisions only from normalized strategy and domain metadata', () => { expect(deriveHostedAvailability({ strategy: 'local', domain: 'localhost' })) .toEqual({ mode: 'local-only', reason: 'local-tool' }); diff --git a/src/hosted/availability.ts b/src/hosted/availability.ts index 3bcdcce0..43ae8f31 100644 --- a/src/hosted/availability.ts +++ b/src/hosted/availability.ts @@ -3,14 +3,16 @@ import { Strategy } from '../registry.js'; export type HostedAvailability = | { mode: 'hosted' } - | { mode: 'local-only'; reason: 'desktop-app' | 'local-tool' | 'browser-bind' }; + | { mode: 'local-only'; reason: 'client-owned' | 'desktop-app' | 'local-tool' | 'browser-bind' }; export interface HostedAvailabilityMetadata { + clientOwned?: boolean; strategy?: Strategy | string; domain?: string; } export function deriveHostedAvailability(command: HostedAvailabilityMetadata): HostedAvailability { + if (command.clientOwned) return { mode: 'local-only', reason: 'client-owned' }; if (String(command.strategy).toLowerCase() === Strategy.LOCAL) { return { mode: 'local-only', reason: 'local-tool' }; } diff --git a/src/hosted/contract.test.ts b/src/hosted/contract.test.ts index 94bc60a2..43843bb0 100644 --- a/src/hosted/contract.test.ts +++ b/src/hosted/contract.test.ts @@ -349,6 +349,22 @@ describe('buildHostedContract', () => { expect(command.keywords).not.toBe(keywords); }); + it('marks client-owned commands local-only while public commands remain hosted', () => { + const contract = buildHostedContract([ + { ...commands[0], name: 'fetch', clientOwned: true }, + { ...commands[0], aliases: undefined }, + ], [], '1.0.0'); + + expect(contract.commands.find(command => command.command === 'web/fetch')).toMatchObject({ + sessionPolicy: 'local-only', + availability: { mode: 'local-only', reason: 'client-owned' }, + }); + expect(contract.commands.find(command => command.command === 'web/profile')).toMatchObject({ + sessionPolicy: 'create-or-reuse', + availability: { mode: 'hosted' }, + }); + }); + it('rejects incomplete file and browser session metadata', () => { const missingDirection = { ...commands[2], diff --git a/src/hosted/contract.ts b/src/hosted/contract.ts index 3d527af5..a5b5d0e2 100644 --- a/src/hosted/contract.ts +++ b/src/hosted/contract.ts @@ -105,6 +105,7 @@ export interface HostedContract { } export interface HostedContractCommandInput { + clientOwned?: boolean; site: string; name: string; aliases?: string[]; diff --git a/src/manifest-types.ts b/src/manifest-types.ts index 22853030..9d35adcf 100644 --- a/src/manifest-types.ts +++ b/src/manifest-types.ts @@ -49,6 +49,8 @@ export interface ManifestEntry { * consumers that load adapters by path must import this export instead. */ packageExport?: string; + /** Command runs only in the local client, never in hosted execution. */ + clientOwned?: boolean; /** Pre-navigation control — see CliCommand.navigateBefore */ navigateBefore?: boolean | string; /** Site session lifecycle defaults — see CliCommand.siteSession */ diff --git a/src/package-exports.test.ts b/src/package-exports.test.ts index 47296c9f..2270e220 100644 --- a/src/package-exports.test.ts +++ b/src/package-exports.test.ts @@ -58,6 +58,36 @@ describe('adapter packaging', () => { expect(fs.existsSync(path.join(ROOT, source)), `${command} export has no source file`).toBe(true); } }); + + it('maps web/fetch to its source and built package export', () => { + const manifest = JSON.parse(fs.readFileSync(path.join(ROOT, 'cli-manifest.json'), 'utf-8')) as Array>; + const command = manifest.find(entry => entry.site === 'web' && entry.name === 'fetch'); + + expect(command).toMatchObject({ packageExport: './fetch/command' }); + expect(fs.existsSync(path.join(ROOT, 'src/fetch/command.ts'))).toBe(true); + expect((pkgJson.exports as Record)['./fetch/command']) + .toBe('./dist/src/fetch/command.js'); + }); + + it('publishes only client-owned web/fetch in generated artifacts', () => { + const manifest = JSON.parse(fs.readFileSync(path.join(ROOT, 'cli-manifest.json'), 'utf-8')) as Array>; + const contract = JSON.parse(fs.readFileSync(path.join(ROOT, 'hosted-contract.json'), 'utf-8')) as { + commands: Array>; + }; + const manifestEntries = manifest.filter(entry => entry.site === 'web'); + const contractEntries = contract.commands.filter(entry => entry.site === 'web'); + + expect(manifestEntries).toEqual([expect.objectContaining({ + name: 'fetch', clientOwned: true, packageExport: './fetch/command', + })]); + expect(manifestEntries[0]).not.toHaveProperty('modulePath'); + expect(manifestEntries[0]).not.toHaveProperty('sourceFile'); + expect(contractEntries).toEqual([expect.objectContaining({ + name: 'fetch', + sessionPolicy: 'local-only', + availability: { mode: 'local-only', reason: 'client-owned' }, + })]); + }); }); describe('package.json exports resolve to real files', () => { From 5ba365b01fa71a45987517cdea2a184f5fdea681 Mon Sep 17 00:00:00 2001 From: beubax Date: Wed, 12 Aug 2026 23:37:25 +0530 Subject: [PATCH 11/24] test(fetch): avoid ignored contract artifact --- src/package-exports.test.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/package-exports.test.ts b/src/package-exports.test.ts index 2270e220..835957ea 100644 --- a/src/package-exports.test.ts +++ b/src/package-exports.test.ts @@ -8,6 +8,7 @@ import { describe, it, expect } from 'vitest'; import * as fs from 'node:fs'; import * as path from 'node:path'; import { fileURLToPath } from 'node:url'; +import { buildManifest, buildManifestArtifacts } from './build-manifest.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(__dirname, '..'); @@ -41,9 +42,8 @@ describe('adapter packaging', () => { expect(pkgJson.scripts.postinstall).not.toMatch(/fetch-adapters/); }); - // webcmd-cloud resolves core-owned commands through `packageExport` rather - // than a clis/ path. A rename that broke this would only surface as a hosted - // runtime failure after publish, so assert the contract here. + // packageExport supports package discovery and import verification for + // core-owned commands; it does not make them Cloud-executable. it('every manifest entry is resolvable: a clis/ path or a real package export', () => { const manifest = JSON.parse(fs.readFileSync(path.join(ROOT, 'cli-manifest.json'), 'utf-8')) as Array>; const exports = pkgJson.exports as Record; @@ -69,9 +69,11 @@ describe('adapter packaging', () => { .toBe('./dist/src/fetch/command.js'); }); - it('publishes only client-owned web/fetch in generated artifacts', () => { - const manifest = JSON.parse(fs.readFileSync(path.join(ROOT, 'cli-manifest.json'), 'utf-8')) as Array>; - const contract = JSON.parse(fs.readFileSync(path.join(ROOT, 'hosted-contract.json'), 'utf-8')) as { + it('publishes only client-owned web/fetch in generated artifacts', async () => { + const { entries } = await buildManifest(); + const artifacts = buildManifestArtifacts(entries, String(pkgJson.version), []); + const manifest = JSON.parse(artifacts.manifestJson) as Array>; + const contract = JSON.parse(artifacts.hostedContractJson) as { commands: Array>; }; const manifestEntries = manifest.filter(entry => entry.site === 'web'); From 234e3e2ef32edc133ac4e8dee58f0064a3729a6e Mon Sep 17 00:00:00 2001 From: beubax Date: Wed, 12 Aug 2026 23:43:40 +0530 Subject: [PATCH 12/24] fix(hosted): present client-owned web fetch locally --- src/completion-shared.ts | 1 + src/hosted/manifest.test.ts | 55 ++++++++++++++++++++++++- src/hosted/manifest.ts | 44 +++++++++++++++++++- src/hosted/root-command-surface.test.ts | 7 ++++ src/hosted/runner.test.ts | 18 ++++++++ src/hosted/runner.ts | 25 +++++++---- src/hosted/types.ts | 1 + 7 files changed, 138 insertions(+), 13 deletions(-) diff --git a/src/completion-shared.ts b/src/completion-shared.ts index f4a41caa..9d51ccdc 100644 --- a/src/completion-shared.ts +++ b/src/completion-shared.ts @@ -45,6 +45,7 @@ export const HOSTED_ROOT_HELP: RootHelpPresentation = { { name: 'list', description: 'List all available hosted CLI commands' }, { name: 'profile', description: 'Manage hosted browser profiles' }, { name: 'setup', description: 'Configure local or hosted mode' }, + { name: 'web', description: 'Fetch URLs locally without launching a browser' }, ], localOnlyCommands: [ { name: 'adapter', description: 'Manage adapters installed on this computer' }, diff --git a/src/hosted/manifest.test.ts b/src/hosted/manifest.test.ts index ab80c9c7..c1c642a2 100644 --- a/src/hosted/manifest.test.ts +++ b/src/hosted/manifest.test.ts @@ -16,6 +16,7 @@ import { renderHostedCommandHelp, renderHostedSiteHelp, siteNames, + withClientOwnedCommands, } from './manifest.js'; import type { HostedManifest } from './types.js'; import { makeHostedConfig } from './config.js'; @@ -87,6 +88,30 @@ function sink(): { stream: Writable; text: () => string } { } describe('hosted manifest helpers', () => { + it('merges the authoritative client-owned web fetch entry exactly once', () => { + const merged = withClientOwnedCommands({ + ...manifest, + commands: [ + ...manifest.commands, + { ...manifest.commands[0]!, site: 'web', name: 'fetch', command: 'web/fetch' }, + ], + }); + + expect(merged.commands.filter(command => command.command === 'web/fetch')).toEqual([ + expect.objectContaining({ + site: 'web', + name: 'fetch', + clientOwned: true, + args: expect.arrayContaining([ + expect.objectContaining({ name: 'url' }), + expect.objectContaining({ name: 'timeout' }), + expect.objectContaining({ name: 'max-chars' }), + expect.objectContaining({ name: 'allow-private' }), + ]), + }), + ]); + }); + it('filters LOCAL commands from hosted list rows', () => { expect(hostedListRows(manifest, true).map((row) => row.command)).toEqual(['github/whoami']); }); @@ -192,7 +217,7 @@ describe('hosted manifest helpers', () => { )); }); - it('uses only executable hosted root capabilities as root completion candidates', async () => { + it('includes client-owned web fetch in hosted discovery surfaces', async () => { const stdout = sink(); await runHostedCli(['--get-completions', '--cursor', '1'], { config: makeHostedConfig({ apiBaseUrl: 'https://api.example.com', apiKey: 'key' }), @@ -200,6 +225,32 @@ describe('hosted manifest helpers', () => { fetchImpl: async () => new Response(JSON.stringify({ ok: true, manifest }), { status: 200 }), }); - expect(stdout.text().trim().split('\n')).toEqual(['browser', 'completion', 'github', 'list', 'profile', 'setup']); + expect(stdout.text().trim().split('\n')).toEqual(['browser', 'completion', 'github', 'list', 'profile', 'setup', 'web']); + + const siteHelp = sink(); + await runHostedCli(['web', '--help'], { + config: makeHostedConfig({ apiBaseUrl: 'https://api.example.com', apiKey: 'key' }), + stdout: siteHelp.stream, + fetchImpl: async () => new Response(JSON.stringify({ ok: true, manifest }), { status: 200 }), + }); + expect(siteHelp.text()).toContain('fetch'); + + const completion = sink(); + await runHostedCli(['--get-completions', '--cursor', '2', 'web'], { + config: makeHostedConfig({ apiBaseUrl: 'https://api.example.com', apiKey: 'key' }), + stdout: completion.stream, + fetchImpl: async () => new Response(JSON.stringify({ ok: true, manifest }), { status: 200 }), + }); + expect(completion.text().trim()).toBe('fetch'); + + const list = sink(); + await runHostedCli(['list', '-f', 'json'], { + config: makeHostedConfig({ apiBaseUrl: 'https://api.example.com', apiKey: 'key' }), + stdout: list.stream, + fetchImpl: async () => new Response(JSON.stringify({ ok: true, manifest }), { status: 200 }), + }); + expect(JSON.parse(list.text())).toEqual(expect.arrayContaining([ + expect.objectContaining({ command: 'web/fetch', clientOwned: true }), + ])); }); }); diff --git a/src/hosted/manifest.ts b/src/hosted/manifest.ts index 50e4f5aa..17a24f84 100644 --- a/src/hosted/manifest.ts +++ b/src/hosted/manifest.ts @@ -11,6 +11,37 @@ import { type CommandListPresentation, } from '../command-presentation.js'; import type { HostedCommand, HostedManifest } from './types.js'; +import { webFetchCommand } from '../fetch/command.js'; + +const clientOwnedCommands: HostedCommand[] = [{ + site: webFetchCommand.site, + name: webFetchCommand.name, + ...(webFetchCommand.aliases ? { aliases: [...webFetchCommand.aliases] } : {}), + command: `${webFetchCommand.site}/${webFetchCommand.name}`, + description: webFetchCommand.description, + access: webFetchCommand.access, + strategy: (webFetchCommand.strategy ?? 'public').toUpperCase(), + browser: webFetchCommand.browser === true, + args: webFetchCommand.args.map(arg => ({ ...arg, ...(arg.choices ? { choices: [...arg.choices] } : {}) })), + columns: [...(webFetchCommand.columns ?? [])], + ...(webFetchCommand.tags ? { tags: [...webFetchCommand.tags] } : {}), + ...(webFetchCommand.keywords ? { keywords: [...webFetchCommand.keywords] } : {}), + ...(webFetchCommand.domain ? { domain: webFetchCommand.domain } : {}), + ...(webFetchCommand.defaultFormat ? { defaultFormat: webFetchCommand.defaultFormat } : {}), + ...(webFetchCommand.freshPage ? { freshPage: true } : {}), + clientOwned: true, +}]; + +export function withClientOwnedCommands(manifest: HostedManifest): HostedManifest { + const localCommandNames = new Set(clientOwnedCommands.map(command => command.command)); + return { + ...manifest, + commands: [ + ...manifest.commands.filter(command => !localCommandNames.has(command.command)), + ...clientOwnedCommands, + ], + }; +} export function isLocalOnlyHostedCommand(command: HostedCommand): boolean { return command.strategy.toUpperCase() === 'LOCAL'; @@ -33,11 +64,20 @@ export function presentHostedCommand(command: HostedCommand): PresentableCommand } export function hostedListRows(manifest: HostedManifest, structured: boolean): Record[] { - return commandListRows(hostedCommands(manifest).map(presentHostedCommand), structured); + return markClientOwned(commandListRows(hostedCommands(manifest).map(presentHostedCommand), structured), structured); } export function hostedListPresentation(manifest: HostedManifest, format: string): CommandListPresentation { - return commandListPresentation(hostedCommands(manifest).map(presentHostedCommand), format); + const presentation = commandListPresentation(hostedCommands(manifest).map(presentHostedCommand), format); + return { + ...presentation, + rows: markClientOwned(presentation.rows, presentation.structured), + }; +} + +function markClientOwned(rows: Record[], structured: boolean): Record[] { + if (!structured) return rows; + return rows.map(row => row.command === 'web/fetch' ? { ...row, clientOwned: true } : row); } export function siteNames(manifest: HostedManifest): string[] { diff --git a/src/hosted/root-command-surface.test.ts b/src/hosted/root-command-surface.test.ts index 6da56199..a3fe4686 100644 --- a/src/hosted/root-command-surface.test.ts +++ b/src/hosted/root-command-surface.test.ts @@ -193,6 +193,13 @@ describe('hosted root command surface', () => { }); }); + it('advertises client-owned web fetch at the hosted root', () => { + expect(HOSTED_ROOT_HELP.commands).toContainEqual({ + name: 'web', + description: 'Fetch URLs locally without launching a browser', + }); + }); + it('advertises hosted profile management as a root command, not a local-only namespace', () => { expect(HOSTED_ROOT_HELP.commands).toContainEqual({ name: 'profile', diff --git a/src/hosted/runner.test.ts b/src/hosted/runner.test.ts index 961c13b1..743dc638 100644 --- a/src/hosted/runner.test.ts +++ b/src/hosted/runner.test.ts @@ -261,6 +261,24 @@ function captureLocalBrowserStructure(argv: string[]): { } describe('runHostedCli', () => { + it('presents web fetch help from local metadata without dispatching it to Cloud', async () => { + const stdout = sink(); + const fetchImpl = vi.fn(async () => manifestResponse()); + + const result = await runHostedCli(['web', 'fetch', '--help'], { + config: makeHostedConfig({ apiBaseUrl: 'https://api.example.com', apiKey: 'key' }), + stdout: stdout.stream, + fetchImpl, + }); + + expect(result).toEqual({ handled: true, exitCode: 0 }); + expect(stdout.text()).toContain('--url '); + expect(stdout.text()).toContain('--timeout [value]'); + expect(stdout.text()).toContain('--max-chars [value]'); + expect(stdout.text()).toContain('--allow-private [value]'); + expect(fetchImpl.mock.calls.map(([url]) => String(url))).toEqual(['https://api.example.com/v1/manifest']); + }); + const publicProfile = { id: 'profile_work', name: 'Work', diff --git a/src/hosted/runner.ts b/src/hosted/runner.ts index f0a62753..00214d1c 100644 --- a/src/hosted/runner.ts +++ b/src/hosted/runner.ts @@ -45,6 +45,7 @@ import { isLocalOnlyHostedCommand, renderHostedCommandHelp, renderHostedSiteHelp, + withClientOwnedCommands, } from './manifest.js'; import { isHostedConfig, loadWebcmdConfig, type WebcmdConfig } from './config.js'; import { resolveHostedApiKey, type HostedCredentialStore } from './credentials.js'; @@ -163,8 +164,7 @@ async function dispatchHosted( return; } if (normalized.kind === 'completion') { - const manifest = await client.getManifest(); - validateManifestContractIdentity(manifest); + const manifest = await getPresentationManifest(client); await writeToStream(stdout, hostedCompletions(manifest, normalized.argv).join('\n') + '\n'); return; } @@ -217,8 +217,7 @@ async function dispatchHosted( await writeToStream(stdout, parsed.output); return; } - const manifest = await client.getManifest(); - validateManifestContractIdentity(manifest); + const manifest = await getPresentationManifest(client); await renderHostedList(manifest, parsed.format, parsed.formatExplicit, stdout, parsed.tag); return; } @@ -341,10 +340,9 @@ async function dispatchHosted( return; } - // The API manifest is tenant-scoped. Never merge package or local plugin - // commands into it: the installed package contract contains no site commands. - const manifest = await client.getManifest(); - validateManifestContractIdentity(manifest); + // The API manifest is tenant-scoped. Only the core client-owned presentation + // entry is merged; package and local plugin commands stay out. + const manifest = await getPresentationManifest(client); const site = args[0]!; const commandName = args[1]; @@ -405,6 +403,9 @@ async function dispatchHosted( await writeHostedHelp(stdout, args, hostedCommandHelpData(command), renderHostedCommandHelp(command)); return; } + if (command.clientOwned) { + throw new Error(`Internal invariant: client-owned command ${command.command} reached hosted dispatch.`); + } const startTime = now(); const response = hasPresentFileArgument(command, parsed.args) @@ -1195,7 +1196,7 @@ function hostedCompletions(manifest: HostedManifest, argv: string[]): string[] { hostedCommands(manifest), words, Number.isFinite(cursor) ? cursor! : words.length, - HOSTED_BUILTIN_COMMANDS, + HOSTED_BUILTIN_COMMANDS.filter(command => command !== 'web'), ); } @@ -1230,6 +1231,12 @@ function validateManifestContractIdentity(manifest: HostedManifest): void { } } +async function getPresentationManifest(client: HostedClient): Promise { + const manifest = await client.getManifest(); + validateManifestContractIdentity(manifest); + return withClientOwnedCommands(manifest); +} + function hostedContractCompatibilityLine(version: string): string | undefined { const match = /^(\d+)\.(\d+)\.(\d+)(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/.exec(version); if (!match) return undefined; diff --git a/src/hosted/types.ts b/src/hosted/types.ts index 6f61c02c..448bfa69 100644 --- a/src/hosted/types.ts +++ b/src/hosted/types.ts @@ -21,6 +21,7 @@ export interface HostedFileArgument { } export interface HostedCommand extends CommandSurfaceMetadata { + clientOwned?: boolean; site: string; name: string; aliases?: string[]; From ae91969ac1033cfe600ab12e3aa6acc64086231a Mon Sep 17 00:00:00 2001 From: beubax Date: Wed, 12 Aug 2026 23:48:29 +0530 Subject: [PATCH 13/24] refactor(fetch): remove browser-backed fetch command --- src/cli.ts | 5 +- src/fetch/browser.test.ts | 440 ---------------- src/fetch/browser.ts | 553 -------------------- src/fetch/command.test.ts | 3 +- src/fetch/command.ts | 24 - src/package-exports.test.ts | 18 + tests/e2e/article-download-pipeline.test.ts | 174 ------ 7 files changed, 21 insertions(+), 1196 deletions(-) delete mode 100644 src/fetch/browser.test.ts delete mode 100644 src/fetch/browser.ts delete mode 100644 tests/e2e/article-download-pipeline.test.ts diff --git a/src/cli.ts b/src/cli.ts index 13fad845..c7cfab03 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -13,9 +13,8 @@ import { fileURLToPath, pathToFileURL } from 'node:url'; import { Command, Option } from 'commander'; import { findPackageRoot, getBuiltEntryCandidates } from './package-paths.js'; import { type CliCommand, getRegistry } from './registry.js'; -// Side-effect import: registers `web fetch` / `web fetch-browser` in the core -// registry so they reach help, `list`, completions and the manifests without a -// plugin install (#252, #247). Both tiers load their implementations lazily. +// Side-effect import: registers client-owned `web fetch` in the core registry +// so it reaches help, `list`, completions and manifests without a plugin. import './fetch/command.js'; import { commandListPresentation, filterCommandsByTag, toPresentableCommand } from './command-presentation.js'; import { configureCompletionCommandSurface, configureListCommandSurface, configurePluginInstallSurface, configurePluginSearchSurface } from './builtin-command-surface.js'; diff --git a/src/fetch/browser.test.ts b/src/fetch/browser.test.ts deleted file mode 100644 index 1c018a95..00000000 --- a/src/fetch/browser.test.ts +++ /dev/null @@ -1,440 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { JSDOM } from 'jsdom'; -import type { IPage } from '../types.js'; -import type { ExtractedPage } from './browser.js'; - -const { mockDownloadArticle } = vi.hoisted(() => ({ - mockDownloadArticle: vi.fn(), -})); - -vi.mock('../download/article-download.js', () => ({ - downloadArticle: mockDownloadArticle, -})); - -const { - buildRenderAwareExtractorJs, - formatDiagnostics, - isInterestingNetworkEntry, - normalizeFrameMode, - normalizeWaitUntil, - runFetchBrowser, -} = await import('./browser.js'); - -describe('web fetch browser tier stdout behavior', () => { - const extractedArticle = { - title: 'Example Article', - author: 'Author', - publishTime: '2026-04-22', - contentHtml: '

      hello

      ', - imageUrls: ['https://example.com/a.jpg'], - diagnostics: { - url: 'https://example.com/article', - frames: [], - emptyContainers: [], - includedFrameCount: 0, - }, - }; - const page = { - goto: vi.fn().mockResolvedValue(undefined), - wait: vi.fn().mockResolvedValue(undefined), - evaluate: vi.fn().mockResolvedValue(extractedArticle), - startNetworkCapture: vi.fn().mockResolvedValue(true), - readNetworkCapture: vi.fn().mockResolvedValue([]), - }; - const run = (kwargs: Record) => runFetchBrowser(page as unknown as IPage, kwargs); - - beforeEach(() => { - vi.restoreAllMocks(); - vi.useRealTimers(); - mockDownloadArticle.mockReset(); - mockDownloadArticle.mockResolvedValue([{ - title: 'Example Article', - author: 'Author', - publish_time: '2026-04-22', - status: 'success', - size: '1 KB', - saved: '-', - }]); - page.goto.mockClear(); - page.wait.mockClear(); - page.evaluate.mockClear(); - page.evaluate.mockResolvedValue(extractedArticle); - page.startNetworkCapture.mockClear(); - page.startNetworkCapture.mockResolvedValue(true); - page.readNetworkCapture.mockClear(); - page.readNetworkCapture.mockResolvedValue([]); - }); - - it('returns null in --stdout mode so the CLI does not append result rows to stdout', async () => { - const result = await run({ - url: 'https://example.com/article', - output: '/tmp/out', - 'download-images': false, - stdout: true, - }); - - expect(result).toBeNull(); - expect(mockDownloadArticle).toHaveBeenCalledWith( - expect.objectContaining({ - title: 'Example Article', - sourceUrl: 'https://example.com/article', - }), - expect.objectContaining({ - output: '/tmp/out', - stdout: true, - }), - ); - expect(page.evaluate.mock.calls[0]?.[0]).toContain('const frameMode = "same-origin"'); - }); - - it('still returns the saved-row payload when writing to disk', async () => { - const rows = [{ title: 'Example Article', saved: '/tmp/out/Example Article/example.md' }]; - mockDownloadArticle.mockResolvedValue(rows); - - const result = await run({ - url: 'https://example.com/article', - output: '/tmp/out', - 'download-images': false, - stdout: false, - }); - - expect(result).toBe(rows); - }); - - it('waits for a selector in the main document or same-origin iframes before extracting', async () => { - page.evaluate - .mockResolvedValueOnce({ ok: true, scope: 'iframe', url: 'https://example.com/frame' }) - .mockResolvedValueOnce(extractedArticle); - - await run({ - url: 'https://example.com/article', - output: '/tmp/out', - 'download-images': false, - 'wait-for': '#gridDatas li', - wait: 7, - stdout: false, - }); - - expect(page.wait).not.toHaveBeenCalled(); - expect(page.evaluate).toHaveBeenCalledTimes(2); - expect(page.evaluate.mock.calls[0]?.[0]).toContain('"#gridDatas li"'); - expect(page.evaluate.mock.calls[0]?.[0]).toContain('sameOriginFrameDocs'); - expect(page.evaluate.mock.calls[1]?.[0]).toContain('const frameMode = "same-origin"'); - }); - - it('throws a clear error when --wait-for times out', async () => { - page.evaluate.mockResolvedValueOnce({ ok: false, timedOut: true, selector: '#missing' }); - - await expect(run({ - url: 'https://example.com/article', - output: '/tmp/out', - 'download-images': false, - 'wait-for': '#missing', - wait: 1, - stdout: false, - })).rejects.toThrow('Timed out waiting for selector "#missing"'); - - expect(mockDownloadArticle).not.toHaveBeenCalled(); - }); - - it('starts network capture and writes diagnostics in diagnose mode', async () => { - const stderr = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); - page.readNetworkCapture.mockResolvedValueOnce([{ - method: 'POST', - url: 'https://example.com/api/data', - responseStatus: 200, - responseContentType: 'application/json', - responsePreview: '{"ok":true}', - }]); - - await run({ - url: 'https://example.com/article', - output: '/tmp/out', - 'download-images': false, - diagnose: true, - wait: 0, - stdout: false, - }); - - expect(page.startNetworkCapture).toHaveBeenCalledWith(''); - expect(page.readNetworkCapture).toHaveBeenCalledTimes(1); - expect(stderr).toHaveBeenCalledWith(expect.stringContaining('[web-fetch-browser diagnose]')); - expect(stderr).toHaveBeenCalledWith(expect.stringContaining('POST 200 application/json https://example.com/api/data')); - }); - - // A challenge interstitial hands off to the real page mid-evaluate. That is - // the normal case on the escalation path, so it must not surface as an error. - it('retries the extractor once when the page navigates out from under it', async () => { - page.evaluate - .mockRejectedValueOnce(new Error('page.evaluate: Execution context was destroyed, most likely because of a navigation')) - .mockResolvedValueOnce(extractedArticle); - - const result = await run({ url: 'https://challenged.example', output: '/tmp/out', 'download-images': false, stdout: false }); - - expect(page.evaluate).toHaveBeenCalledTimes(2); - expect(mockDownloadArticle).toHaveBeenCalledOnce(); - expect(result).not.toBeNull(); - }); - - it('rethrows a second navigation failure instead of looping', async () => { - const destroyed = new Error('Execution context was destroyed'); - page.evaluate.mockRejectedValueOnce(destroyed).mockRejectedValueOnce(destroyed); - - await expect(run({ url: 'https://challenged.example', output: '/tmp/out', 'download-images': false, stdout: false })) - .rejects.toThrow('Execution context was destroyed'); - expect(page.evaluate).toHaveBeenCalledTimes(2); - }); - - it('does not retry an unrelated evaluate failure', async () => { - page.evaluate.mockRejectedValueOnce(new Error('SyntaxError: bad selector')); - - await expect(run({ url: 'https://example.com/article', output: '/tmp/out', 'download-images': false, stdout: false })) - .rejects.toThrow('SyntaxError'); - expect(page.evaluate).toHaveBeenCalledTimes(1); - }); - - it('passes --frames none into the extractor', async () => { - await run({ - url: 'https://example.com/article', - output: '/tmp/out', - 'download-images': false, - frames: 'none', - stdout: false, - }); - - expect(page.evaluate.mock.calls[0]?.[0]).toContain('const frameMode = "none"'); - }); - - it('passes --frames all-same-origin into the extractor', async () => { - await run({ - url: 'https://example.com/article', - output: '/tmp/out', - 'download-images': false, - frames: 'all-same-origin', - stdout: false, - }); - - expect(page.evaluate.mock.calls[0]?.[0]).toContain('const frameMode = "all-same-origin"'); - }); - - it('fails fast when --wait-until networkidle is requested but capture is unavailable', async () => { - page.startNetworkCapture.mockResolvedValue(false); - - await expect(run({ - url: 'https://example.com/article', - output: '/tmp/out', - 'download-images': false, - 'wait-until': 'networkidle', - wait: 2, - stdout: false, - })).rejects.toThrow('Network capture is unavailable'); - - expect(page.wait).not.toHaveBeenCalled(); - expect(mockDownloadArticle).not.toHaveBeenCalled(); - }); - - it('fails fast when network traffic never settles before the networkidle timeout', async () => { - vi.useFakeTimers(); - page.readNetworkCapture.mockResolvedValue([{ - method: 'POST', - url: 'https://example.com/api/data', - responseStatus: 200, - responseContentType: 'application/json', - responsePreview: '{"ok":true}', - }]); - - const pending = expect(run({ - url: 'https://example.com/article', - output: '/tmp/out', - 'download-images': false, - 'wait-until': 'networkidle', - wait: 1, - stdout: false, - })).rejects.toThrow('Timed out waiting for network idle after 1s'); - - await vi.advanceTimersByTimeAsync(2000); - - await pending; - expect(mockDownloadArticle).not.toHaveBeenCalled(); - }); -}); - -describe('web fetch browser tier render-aware helpers', () => { - it('merges accessible same-origin iframe bodies into the extracted HTML', () => { - const dom = new JSDOM(` -
      -

      Shell

      - -
      - `, { url: 'https://example.com/main.html', runScripts: 'outside-only' }); - const frame = dom.window.document.querySelector('iframe')!; - frame.contentDocument!.open(); - frame.contentDocument!.write('
      Name

        Station A 42

        '); - frame.contentDocument!.close(); - - const result = dom.window.eval(buildRenderAwareExtractorJs({ frames: 'same-origin' })) as ExtractedPage; - - expect(result.diagnostics.includedFrameCount).toBe(1); - expect(result.contentHtml).toContain('data-webcmd-iframe-source="https://example.com/frame.html"'); - expect(result.contentHtml).toContain('localized text iframe: https://example.com/frame.html'); - expect(result.contentHtml).toContain('Station A 42'); - expect(result.diagnostics.emptyContainers).toEqual(expect.arrayContaining([ - expect.objectContaining({ scope: 'iframe', id: 'gridDatas', url: 'https://example.com/frame.html' }), - ])); - expect(result.diagnostics.emptyContainers.every((item: { scope: string }) => item.scope === 'iframe')).toBe(true); - }); - - it('merges readable same-origin iframes outside the selected content element', () => { - const dom = new JSDOM(` -
        -

        Main Article

        -

        ${'Main content '.repeat(30)}

        -
        - - `, { url: 'https://example.com/main.html', runScripts: 'outside-only' }); - const frame = dom.window.document.querySelector('iframe')!; - frame.contentDocument!.open(); - frame.contentDocument!.write(`

        Outside Frame

        ${'Frame data '.repeat(12)}

        `); - frame.contentDocument!.close(); - - const result = dom.window.eval(buildRenderAwareExtractorJs({ frames: 'same-origin' })) as ExtractedPage; - - expect(result.diagnostics.includedFrameCount).toBe(1); - expect(result.contentHtml).toContain('data-webcmd-iframe-source="https://example.com/outside.html"'); - expect(result.contentHtml).toContain('Outside Frame'); - expect(result.contentHtml).toContain('Frame data'); - }); - - it('keeps short data-like iframes outside the selected content element', () => { - const dom = new JSDOM(` -
        -

        Main Article

        -

        ${'Main content '.repeat(30)}

        -
        - - `, { url: 'https://example.com/main.html', runScripts: 'outside-only' }); - const frame = dom.window.document.querySelector('iframe')!; - frame.contentDocument!.open(); - frame.contentDocument!.write('
        water level
        42
          '); - frame.contentDocument!.close(); - - const result = dom.window.eval(buildRenderAwareExtractorJs({ frames: 'same-origin' })) as ExtractedPage; - - expect(result.diagnostics.includedFrameCount).toBe(1); - expect(result.contentHtml).toContain('42'); - expect(result.diagnostics.emptyContainers).toEqual(expect.arrayContaining([ - expect.objectContaining({ scope: 'iframe', id: 'gridDatas', url: 'https://example.com/data.html' }), - ])); - expect(result.diagnostics.emptyContainers.every((item: { scope: string }) => item.scope === 'iframe')).toBe(true); - }); - - it('skips short non-structural iframes outside the selected content element', () => { - const dom = new JSDOM(` -
          -

          Main Article

          -

          ${'Main content '.repeat(30)}

          -
          - - `, { url: 'https://example.com/main.html', runScripts: 'outside-only' }); - const frame = dom.window.document.querySelector('iframe')!; - frame.contentDocument!.open(); - frame.contentDocument!.write('

          tiny note

          '); - frame.contentDocument!.close(); - - const result = dom.window.eval(buildRenderAwareExtractorJs({ frames: 'same-origin' })) as ExtractedPage; - - expect(result.diagnostics.includedFrameCount).toBe(0); - expect(result.contentHtml).not.toContain('tiny note'); - }); - - it('includes short non-structural iframes in all-same-origin mode', () => { - const dom = new JSDOM(` -
          -

          Main Article

          -

          ${'Main content '.repeat(30)}

          -
          - - `, { url: 'https://example.com/main.html', runScripts: 'outside-only' }); - const frame = dom.window.document.querySelector('iframe')!; - frame.contentDocument!.open(); - frame.contentDocument!.write('
          Online: 42°C
          '); - frame.contentDocument!.close(); - - const result = dom.window.eval(buildRenderAwareExtractorJs({ frames: 'all-same-origin' })) as ExtractedPage; - - expect(result.diagnostics.includedFrameCount).toBe(1); - expect(result.contentHtml).toContain('Online: 42°C'); - }); - - it('marks API-like network entries as interesting and ignores static assets', () => { - expect(isInterestingNetworkEntry({ - method: 'POST', - url: 'https://example.com/GJZ/Ajax/Publish.ashx', - status: 200, - contentType: 'text/html', - size: 100, - bodyTruncated: false, - })).toBe(true); - expect(isInterestingNetworkEntry({ - method: 'POST', - url: 'https://example.com/GJZ/Ajax/Publish.ashx', - status: 200, - contentType: 'application/json', - size: 100, - bodyTruncated: false, - })).toBe(true); - expect(isInterestingNetworkEntry({ - method: 'GET', - url: 'https://example.com/app.js', - status: 200, - contentType: 'application/javascript', - size: 100, - bodyTruncated: false, - })).toBe(false); - }); - - it('formats frame and XHR diagnostics for shell pages', () => { - const output = formatDiagnostics({ - diagnostics: { - url: 'https://example.com/main.html', - includedFrameCount: 1, - frames: [{ - index: 0, - src: 'https://example.com/frame.html', - title: '', - sameOrigin: true, - accessible: true, - textLength: 42, - }], - emptyContainers: [{ - scope: 'iframe', - url: 'https://example.com/frame.html', - tag: 'ul', - id: 'gridDatas', - className: '', - }], - }, - }, [{ - method: 'POST', - url: 'https://example.com/GJZ/Ajax/Publish.ashx', - status: 200, - contentType: 'application/json', - size: 64, - bodyTruncated: false, - }], true); - - expect(output).toContain('frames: 1, included_same_origin: 1'); - expect(output).toContain('[frame 0] same-origin accessible text=42 https://example.com/frame.html'); - expect(output).toContain('iframe: ul#gridDatas (https://example.com/frame.html)'); - expect(output).toContain('POST 200 application/json https://example.com/GJZ/Ajax/Publish.ashx'); - }); - - it('normalizes unknown frame and wait-until values to their defaults', () => { - expect(normalizeFrameMode('bogus')).toBe('same-origin'); - expect(normalizeFrameMode('none')).toBe('none'); - expect(normalizeWaitUntil('bogus')).toBe('domstable'); - expect(normalizeWaitUntil('networkidle')).toBe('networkidle'); - }); -}); diff --git a/src/fetch/browser.ts b/src/fetch/browser.ts deleted file mode 100644 index ccbc7c10..00000000 --- a/src/fetch/browser.ts +++ /dev/null @@ -1,553 +0,0 @@ -/** - * Browser tier of `webcmd web fetch`. - * - * Renders a page in a real browser and extracts the main content with DOM - * heuristics: - * 1.
          element - * 2. [role="main"] element - * 3.
          element - * 4. Largest text-dense block as fallback - * - * Pipes through the shared article-download pipeline (Turndown + image download). - * - * This is the escalation target when the plain and impit tiers in `client.ts` - * cannot read a page. It moved here from `clis/web/fetch-browser.js` so the - * whole fetch ladder ships in the core package rather than an adapter that was - * never installed by default (#247). - */ -import { articleHtmlToMarkdown, downloadArticle } from '../download/article-download.js'; -import type { CommandArgs } from '../registry.js'; -import type { IPage } from '../types.js'; - -export const DEFAULT_OUTPUT_DIR = './web-articles'; - -const NETWORK_IDLE_QUIET_MS = 1000; -const NETWORK_IDLE_POLL_MS = 500; -const MIN_NON_STRUCTURAL_IFRAME_TEXT_CHARS = 50; - -export type FrameMode = 'same-origin' | 'all-same-origin' | 'none'; -export type WaitUntil = 'domstable' | 'networkidle'; - -export interface NetworkEntry { - method: string; - url: string; - status: number; - contentType: string; - size: number; - bodyTruncated: boolean; -} - -export interface FrameDiagnostic { - index: number; - src: string; - title: string; - sameOrigin: boolean; - accessible: boolean; - textLength: number; -} - -export interface ExtractedPage { - title: string; - author: string; - publishTime: string; - contentHtml: string; - imageUrls: string[]; - diagnostics: { - url: string; - frames: FrameDiagnostic[]; - emptyContainers: Array<{ scope: string; url: string; tag: string; id: string; className: string }>; - includedFrameCount: number; - }; -} - -function sleep(ms: number): Promise { - return new Promise(resolve => setTimeout(resolve, ms)); -} - -export function boolish(value: unknown): boolean { - if (value === true) return true; - if (typeof value === 'string') return ['1', 'true', 'yes', 'on'].includes(value.toLowerCase()); - return false; -} - -export function normalizeFrameMode(value: unknown): FrameMode { - const mode = String(value || 'same-origin').toLowerCase(); - if (mode === 'same-origin' || mode === 'all-same-origin' || mode === 'none') return mode; - return 'same-origin'; -} - -export function normalizeWaitUntil(value: unknown): WaitUntil { - const waitUntil = String(value || 'domstable').toLowerCase(); - if (waitUntil === 'domstable' || waitUntil === 'networkidle') return waitUntil; - return 'domstable'; -} - -function normalizeNetworkEntry(entry: Record | null | undefined): NetworkEntry { - const preview = typeof entry?.responsePreview === 'string' ? entry.responsePreview : ''; - return { - method: typeof entry?.method === 'string' ? entry.method : 'GET', - url: typeof entry?.url === 'string' ? entry.url : '', - status: typeof entry?.responseStatus === 'number' ? entry.responseStatus : 0, - contentType: typeof entry?.responseContentType === 'string' ? entry.responseContentType : '', - size: typeof entry?.responseBodyFullSize === 'number' ? entry.responseBodyFullSize : preview.length, - bodyTruncated: entry?.responseBodyTruncated === true, - }; -} - -export function isInterestingNetworkEntry(entry: NetworkEntry): boolean { - const ct = (entry.contentType || '').toLowerCase(); - const url = entry.url || ''; - const method = (entry.method || 'GET').toUpperCase(); - const staticAsset = /\.(js|css|png|jpg|jpeg|gif|svg|woff|woff2|ico|map)(\?|$)/i.test(url); - const noisy = /analytics|tracking|telemetry|beacon|pixel|gtag|fbevents/i.test(url); - const apiLikeUrl = /\/(api|ajax|graphql|rest|service|handler)(\/|[?._-]|$)|\.(ashx|aspx|asmx|php)(\?|$)/i.test(url); - const dataLikeContent = ct.includes('json') - || ct.includes('xml') - || ct.includes('text/plain') - || ct.includes('javascript') - || (apiLikeUrl && ct.includes('text/html')); - return ( - !staticAsset - && !noisy - && (dataLikeContent || apiLikeUrl || method !== 'GET') - ); -} - -async function drainNetworkCapture(page: IPage, sink: NetworkEntry[]): Promise { - if (!page.readNetworkCapture) return []; - const raw = await page.readNetworkCapture().catch(() => []); - const entries = Array.isArray(raw) - ? raw.map(entry => normalizeNetworkEntry(entry as Record)).filter(entry => entry.url) - : []; - sink.push(...entries); - return entries; -} - -async function maybeStartNetworkCapture(page: IPage): Promise { - if (!page.startNetworkCapture) return false; - try { - return await page.startNetworkCapture(''); - } catch { - return false; - } -} - -async function waitForNetworkIdle(page: IPage, maxSeconds: number, sink: NetworkEntry[]): Promise<{ ok: boolean; timedOut?: boolean }> { - const timeoutMs = Math.max(1, Number(maxSeconds) || 1) * 1000; - const deadline = Date.now() + timeoutMs; - let quietSince = Date.now(); - while (Date.now() < deadline) { - const entries = await drainNetworkCapture(page, sink); - if (entries.length > 0) quietSince = Date.now(); - if (Date.now() - quietSince >= NETWORK_IDLE_QUIET_MS) return { ok: true }; - await sleep(NETWORK_IDLE_POLL_MS); - } - return { ok: false, timedOut: true }; -} - -export function buildWaitForSelectorAcrossFramesJs(selector: string, timeoutMs: number): string { - return ` - (async () => { - const selector = ${JSON.stringify(selector)}; - const timeoutAt = Date.now() + ${Number(timeoutMs) || 10000}; - const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)); - const sameOriginFrameDocs = () => Array.from(document.querySelectorAll('iframe')).map((frame) => { - try { - const href = new URL(frame.getAttribute('src') || frame.src || '', window.location.href).href; - if (new URL(href).origin !== window.location.origin) return null; - return { href, doc: frame.contentDocument }; - } catch { - return null; - } - }).filter(Boolean); - const findMatch = () => { - try { - if (document.querySelector(selector)) return { ok: true, scope: 'main', url: window.location.href }; - } catch (err) { - return { ok: false, invalidSelector: true, error: String(err && err.message || err) }; - } - for (const frame of sameOriginFrameDocs()) { - try { - if (frame.doc?.querySelector(selector)) return { ok: true, scope: 'iframe', url: frame.href }; - } catch {} - } - return { ok: false }; - }; - while (Date.now() < timeoutAt) { - const found = findMatch(); - if (found.ok || found.invalidSelector) return found; - await sleep(100); - } - return { ok: false, timedOut: true, selector }; - })() - `; -} - -export function buildRenderAwareExtractorJs(options: { frames: FrameMode }): string { - return ` - (() => { - const frameMode = ${JSON.stringify(options.frames)}; - const minNonStructuralIframeTextChars = ${MIN_NON_STRUCTURAL_IFRAME_TEXT_CHARS}; - const result = { - title: '', - author: '', - publishTime: '', - contentHtml: '', - imageUrls: [], - diagnostics: { - url: window.location.href, - frames: [], - emptyContainers: [], - includedFrameCount: 0 - } - }; - - const absolutize = (value, base) => { - if (!value || value.startsWith('data:') || value.startsWith('javascript:') || value.startsWith('#')) return value || ''; - try { return new URL(value, base).href; } catch { return value; } - }; - const absolutizeTree = (root, base) => { - root.querySelectorAll?.('[href]').forEach(el => el.setAttribute('href', absolutize(el.getAttribute('href'), base))); - root.querySelectorAll?.('[src]').forEach(el => el.setAttribute('src', absolutize(el.getAttribute('src'), base))); - root.querySelectorAll?.('[poster]').forEach(el => el.setAttribute('poster', absolutize(el.getAttribute('poster'), base))); - root.querySelectorAll?.('[action]').forEach(el => el.setAttribute('action', absolutize(el.getAttribute('action'), base))); - }; - const textLen = (node) => (node?.textContent || '').replace(/\\s+/g, ' ').trim().length; - const describeFrame = (frame, index) => { - const rawSrc = frame.getAttribute('src') || frame.src || ''; - let href = ''; - try { href = new URL(rawSrc, window.location.href).href; } catch { href = rawSrc; } - let sameOrigin = false; - try { sameOrigin = href ? new URL(href).origin === window.location.origin : false; } catch {} - let accessible = false; - let title = frame.getAttribute('title') || frame.getAttribute('name') || frame.id || ''; - let length = 0; - try { - accessible = !!frame.contentDocument; - title = title || frame.contentDocument?.title || ''; - length = textLen(frame.contentDocument?.body); - } catch {} - return { index, src: href, title, sameOrigin, accessible, textLength: length }; - }; - const collectEmptyContainers = (root, scope, baseUrl) => { - const likely = 'table, tbody, ul[id], ol[id], div[id], section[id], [class*="grid"], [class*="data"], [class*="list"], [id*="grid"], [id*="data"], [id*="list"]'; - root.querySelectorAll?.(likely).forEach((el) => { - if (scope === 'main' && el.closest?.('[data-webcmd-iframe-source]')) return; - const id = el.getAttribute('id') || ''; - const cls = el.getAttribute('class') || ''; - const name = [id, cls].join(' ').toLowerCase(); - if (!/(grid|data|list|table|content|result)/.test(name) && !['TABLE', 'TBODY', 'UL', 'OL'].includes(el.nodeName)) return; - if (textLen(el) > 20) return; - result.diagnostics.emptyContainers.push({ - scope, - url: baseUrl, - tag: el.tagName.toLowerCase(), - id, - className: cls, - }); - }); - }; - const hasDataContainerSignal = (root) => { - const likely = 'table, tbody, ul[id], ol[id], [id*="grid"], [id*="data"], [id*="list"], [id*="content"], [id*="result"], [class*="grid"], [class*="data"], [class*="list"], [class*="content"], [class*="result"]'; - return !!root.querySelector?.(likely); - }; - const shouldIncludeExternalFrame = (frameBody) => { - // Outside-content iframes are less trusted than placeholders inside - // contentEl. Long plain text is the fallback for simple same-origin - // frames that lack article/table/list structure. - if (textLen(frameBody) >= minNonStructuralIframeTextChars) return true; - if (frameBody.querySelector?.('article, main, [role="main"], table, tbody, ul li, ol li')) return true; - return hasDataContainerSignal(frameBody); - }; - const buildFrameSection = (frameBody, desc, fallbackLabel) => { - absolutizeTree(frameBody, desc.src || window.location.href); - collectEmptyContainers(frameBody, 'iframe', desc.src); - const section = document.createElement('section'); - section.setAttribute('data-webcmd-iframe-source', desc.src); - const heading = document.createElement('h2'); - heading.textContent = 'localized text iframe: ' + (desc.src || fallbackLabel); - section.appendChild(heading); - Array.from(frameBody.childNodes).forEach(node => section.appendChild(node)); - return section; - }; - - const ogTitle = document.querySelector('meta[property="og:title"]'); - if (ogTitle) result.title = ogTitle.getAttribute('content')?.trim() || ''; - if (!result.title) result.title = document.title?.trim() || ''; - if (!result.title) result.title = document.querySelector('h1')?.textContent?.trim() || 'untitled'; - result.title = result.title.replace(/\\s*[|\\-–—]\\s*[^|\\-–—]{1,30}$/, '').trim(); - - const authorMeta = document.querySelector('meta[name="author"], meta[property="article:author"], meta[name="twitter:creator"]'); - result.author = authorMeta?.getAttribute('content')?.trim() || ''; - - const timeMeta = document.querySelector('meta[property="article:published_time"], meta[name="date"], meta[name="publishdate"], time[datetime]'); - if (timeMeta) { - result.publishTime = timeMeta.getAttribute('content') - || timeMeta.getAttribute('datetime') - || timeMeta.textContent?.trim() - || ''; - } - - let contentEl = null; - const articles = document.querySelectorAll('article'); - if (articles.length === 1) { - contentEl = articles[0]; - } else if (articles.length > 1) { - let maxLen = 0; - articles.forEach(a => { - const len = textLen(a); - if (len > maxLen) { maxLen = len; contentEl = a; } - }); - } - if (!contentEl) contentEl = document.querySelector('[role="main"]'); - if (!contentEl) contentEl = document.querySelector('main'); - if (!contentEl) { - const candidates = document.querySelectorAll( - 'div[class*="content"], div[class*="article"], div[class*="post"], ' + - 'div[class*="entry"], div[class*="body"], div[id*="content"], ' + - 'div[id*="article"], div[id*="post"], section' - ); - let maxLen = 0; - candidates.forEach(c => { - const len = textLen(c); - if (len > maxLen) { maxLen = len; contentEl = c; } - }); - } - if (!contentEl || textLen(contentEl) < 200) contentEl = document.body; - - const clone = contentEl.cloneNode(true); - absolutizeTree(clone, window.location.href); - - const originalFrames = Array.from(contentEl.querySelectorAll('iframe')); - const clonedFrames = Array.from(clone.querySelectorAll('iframe')); - const clonedFrameByOriginal = new Map(); - originalFrames.forEach((frame, index) => { - const cloned = clonedFrames[index]; - if (cloned) clonedFrameByOriginal.set(frame, cloned); - }); - const allFrames = Array.from(document.querySelectorAll('iframe')); - const frameDescriptions = new Map(); - allFrames.forEach((frame, index) => frameDescriptions.set(frame, describeFrame(frame, index))); - const getFrameDescription = (frame, fallbackIndex) => frameDescriptions.get(frame) || describeFrame(frame, fallbackIndex); - result.diagnostics.frames = allFrames.map(frame => frameDescriptions.get(frame)); - - if (frameMode === 'same-origin' || frameMode === 'all-same-origin') { - allFrames.forEach((frame, index) => { - const insideContent = contentEl.contains(frame); - const cloned = insideContent ? clonedFrameByOriginal.get(frame) : null; - if (insideContent && !cloned) return; - const desc = getFrameDescription(frame, index); - if (!desc.sameOrigin || !desc.accessible) return; - try { - const doc = frame.contentDocument; - if (!doc?.body) return; - const frameBody = doc.body.cloneNode(true); - if (frameMode !== 'all-same-origin' && !insideContent && !shouldIncludeExternalFrame(frameBody)) return; - const section = buildFrameSection(frameBody, desc, frame.getAttribute('src') || ('#' + index)); - if (insideContent) cloned.replaceWith(section); - else clone.appendChild(section); - result.diagnostics.includedFrameCount += 1; - } catch {} - }); - } - - collectEmptyContainers(clone, 'main', window.location.href); - - const noise = 'nav, header, footer, aside, .sidebar, .nav, .menu, .footer, ' + - '.header, .comments, .comment, .ad, .ads, .advertisement, .social-share, ' + - '.related-posts, .newsletter, .cookie-banner, script, style, noscript, iframe'; - clone.querySelectorAll(noise).forEach(el => el.remove()); - - const stripWS = (s) => (s || '').replace(/\\s+/g, ''); - const dedup = (parent) => { - const children = Array.from(parent.children || []); - for (let i = children.length - 1; i >= 1; i--) { - const curRaw = children[i].textContent || ''; - const prevRaw = children[i - 1].textContent || ''; - const cur = stripWS(curRaw); - const prev = stripWS(prevRaw); - if (cur.length < 20 || prev.length < 20) continue; - if (cur === prev) { - const curSpaces = (curRaw.match(/ /g) || []).length; - const prevSpaces = (prevRaw.match(/ /g) || []).length; - if (curSpaces >= prevSpaces) children[i - 1].remove(); - else children[i].remove(); - } else if (prev.includes(cur) && cur.length / prev.length > 0.8) { - children[i].remove(); - } else if (cur.includes(prev) && prev.length / cur.length > 0.8) { - children[i - 1].remove(); - } - } - }; - dedup(clone); - clone.querySelectorAll('section, div').forEach(el => { - if (el.children && el.children.length > 2) dedup(el); - }); - - clone.querySelectorAll('img').forEach(img => { - const srcset = img.getAttribute('data-srcset') || ''; - const srcsetFirst = srcset.split(',')[0]?.trim().split(' ')[0] || ''; - const real = img.getAttribute('data-src') - || img.getAttribute('data-original') - || img.getAttribute('data-lazy-src') - || srcsetFirst; - if (real) img.setAttribute('src', absolutize(real, window.location.href)); - }); - - result.contentHtml = clone.innerHTML; - - const seen = new Set(); - clone.querySelectorAll('img').forEach(img => { - const src = img.getAttribute('src') || ''; - if (src && !src.startsWith('data:') && !seen.has(src)) { - seen.add(src); - result.imageUrls.push(src); - } - }); - - return result; - })() - `; -} - -/** - * A page that navigates after load — a challenge interstitial handing off to - * the real page, a client-side redirect — destroys the execution context out - * from under `evaluate`. That is the normal case on the escalation path, since - * escalation only happens for pages that blocked plain HTTP in the first place. - * Retry once after letting the new document settle; a second failure is real. - */ -async function evaluateAfterNavigation(page: IPage, js: string, settleSeconds: number): Promise { - try { - return await page.evaluate(js); - } catch (error) { - if (!/execution context was destroyed|context was destroyed|navigation/i.test(String((error as Error)?.message ?? error))) throw error; - await page.wait(Math.max(1, settleSeconds)); - return page.evaluate(js); - } -} - -export function formatDiagnostics( - data: Partial | null | undefined, - networkEntries: NetworkEntry[], - captureSupported: boolean, -): string { - const lines: string[] = []; - const diag = data?.diagnostics ?? ({} as Partial); - lines.push('[web-fetch-browser diagnose]'); - lines.push(`url: ${diag.url || '-'}`); - lines.push(`frames: ${Array.isArray(diag.frames) ? diag.frames.length : 0}, included_same_origin: ${diag.includedFrameCount || 0}`); - for (const frame of (diag.frames || []).slice(0, 20)) { - lines.push(` [frame ${frame.index}] ${frame.sameOrigin ? 'same-origin' : 'cross-origin'} ${frame.accessible ? 'accessible' : 'blocked'} text=${frame.textLength || 0} ${frame.src || '-'}`); - } - if (Array.isArray(diag.emptyContainers) && diag.emptyContainers.length > 0) { - lines.push(`empty_containers: ${diag.emptyContainers.length}`); - for (const item of diag.emptyContainers.slice(0, 12)) { - const selector = `${item.tag}${item.id ? `#${item.id}` : ''}${item.className ? `.${String(item.className).trim().split(/\s+/).filter(Boolean).join('.')}` : ''}`; - lines.push(` ${item.scope}: ${selector} (${item.url || '-'})`); - } - } - const interesting = networkEntries.filter(isInterestingNetworkEntry); - lines.push(`network_capture: ${captureSupported ? 'enabled' : 'unavailable'}, entries=${networkEntries.length}, api_like=${interesting.length}`); - for (const entry of interesting.slice(0, 20)) { - lines.push(` ${entry.method} ${entry.status || '-'} ${entry.contentType || '-'} ${entry.url}`); - } - return `${lines.join('\n')}\n`; -} - -/** - * Render `--url` in the browser and return its main content as markdown. - * - * This is the escalation target for `web fetch`: it deliberately writes no - * files and downloads no images, so an escalated fetch answers in the same - * shape as the plain and impit tiers and still honours `-f`. The file-export - * pipeline stays behind `web fetch-browser`. - */ -export async function extractPageMarkdown(page: IPage, kwargs: CommandArgs): Promise<{ title: string; content: string }> { - const url = String(kwargs.url); - const waitSeconds = Number(kwargs.wait ?? 3); - await page.goto(url); - await page.wait(waitSeconds); - const data = await evaluateAfterNavigation( - page, - buildRenderAwareExtractorJs({ frames: normalizeFrameMode(kwargs.frames) }), - waitSeconds, - ); - return { - title: data?.title || '', - content: articleHtmlToMarkdown(data?.contentHtml || ''), - }; -} - -/** - * Render `--url` in the browser and hand the extracted article to the shared - * download pipeline. Returns `null` in `--stdout` mode: the markdown body has - * already gone to process.stdout inside downloadArticle(), so returning rows - * would make Commander append table/JSON output to the same stream and break - * piping. - */ -export async function runFetchBrowser(page: IPage, kwargs: CommandArgs, debug: boolean = false): Promise { - const url = String(kwargs.url); - const waitSeconds = Number(kwargs.wait ?? 3); - const waitUntil = normalizeWaitUntil(kwargs['wait-until']); - const frameMode = normalizeFrameMode(kwargs.frames); - const shouldDiagnose = boolish(kwargs.diagnose) || debug || !!process.env.WEBCMD_VERBOSE; - const networkEntries: NetworkEntry[] = []; - const captureSupported = (waitUntil === 'networkidle' || shouldDiagnose) - ? await maybeStartNetworkCapture(page) - : false; - // Navigate to the target URL - await page.goto(url); - if (kwargs['wait-for']) { - const waitResult = await page.evaluate<{ ok?: boolean; invalidSelector?: boolean; error?: string }>( - buildWaitForSelectorAcrossFramesJs(String(kwargs['wait-for']), waitSeconds * 1000), - ); - if (waitResult?.invalidSelector) { - throw new Error(`Invalid --wait-for selector "${kwargs['wait-for']}": ${waitResult.error || 'querySelector failed'}`); - } - if (!waitResult?.ok) { - throw new Error(`Timed out waiting for selector "${kwargs['wait-for']}" in main document or same-origin iframes`); - } - } else if (waitUntil !== 'networkidle') { - await page.wait(waitSeconds); - } - if (waitUntil === 'networkidle') { - if (!captureSupported) { - throw new Error('Network capture is unavailable, so --wait-until networkidle cannot be satisfied'); - } - const idle = await waitForNetworkIdle(page, waitSeconds, networkEntries); - if (!idle?.ok) { - throw new Error(`Timed out waiting for network idle after ${waitSeconds}s`); - } - } - // Extract article content using browser-side heuristics - const data = await evaluateAfterNavigation(page, buildRenderAwareExtractorJs({ frames: frameMode }), waitSeconds); - if (captureSupported) await drainNetworkCapture(page, networkEntries); - if (shouldDiagnose) process.stderr.write(formatDiagnostics(data, networkEntries, captureSupported)); - // Determine Referer from URL for image downloads - let referer = ''; - try { - const parsed = new URL(url); - referer = `${parsed.origin}/`; - } catch { /* ignore */ } - const result = await downloadArticle({ - title: data?.title || 'untitled', - author: data?.author, - publishTime: data?.publishTime, - sourceUrl: url, - contentHtml: data?.contentHtml || '', - imageUrls: data?.imageUrls, - }, { - output: String(kwargs.output ?? DEFAULT_OUTPUT_DIR), - downloadImages: kwargs['download-images'] !== false, - imageHeaders: referer ? { Referer: referer } : undefined, - stdout: kwargs.stdout === true, - configureTurndown: (td) => { - td.addRule('preserveButtons', { - filter: (node) => node.nodeName === 'BUTTON', - replacement: (content) => content, - }); - }, - }); - return kwargs.stdout ? null : result; -} diff --git a/src/fetch/command.test.ts b/src/fetch/command.test.ts index 49d66e0b..493d7231 100644 --- a/src/fetch/command.test.ts +++ b/src/fetch/command.test.ts @@ -13,7 +13,7 @@ vi.mock('../output.js', async () => ({ })); import { registerCommandToProgram } from '../commanderAdapter.js'; -import { formatWebFetchMarkdown, webFetchBrowserCommand, webFetchCommand } from './command.js'; +import { formatWebFetchMarkdown, webFetchCommand } from './command.js'; const plainResult = { status: 200, requestedUrl: 'https://example.com', finalUrl: 'https://example.com', contentType: 'text/plain', tier: 'plain' as const, title: 'Example', extractionSource: 'raw' as const, truncated: false, content: 'ok' }; @@ -32,7 +32,6 @@ describe('web fetch command', () => { it('is the client-owned, non-browser core command', () => { expect(webFetchCommand).toMatchObject({ site: 'web', name: 'fetch', browser: false, clientOwned: true, defaultFormat: 'md' }); - expect(webFetchBrowserCommand.name).toBe('fetch-browser'); }); it('uses Commander coercion for canonical fetch options', async () => { diff --git a/src/fetch/command.ts b/src/fetch/command.ts index a66e166e..41966f62 100644 --- a/src/fetch/command.ts +++ b/src/fetch/command.ts @@ -2,30 +2,6 @@ import { cli, Strategy, type CommandArgs } from '../registry.js'; import { ArgumentError } from '../errors.js'; import type { WebFetchOptions, WebFetchResult } from './client.js'; -/** Kept in sync with `DEFAULT_OUTPUT_DIR` in ./browser.js, which is imported - * lazily so Turndown and the download pipeline stay out of CLI startup. */ -const DEFAULT_OUTPUT_DIR = './web-articles'; - -export const webFetchBrowserCommand = cli({ - site: 'web', name: 'fetch-browser', access: 'read', - description: 'Fetch any web page in a real browser and export as Markdown', - strategy: Strategy.COOKIE, - navigateBefore: false, // we handle navigation ourselves - args: [ - { name: 'url', required: true, help: 'Any web page URL' }, - { name: 'output', default: DEFAULT_OUTPUT_DIR, help: 'Output directory' }, - { name: 'download-images', type: 'boolean', default: true, help: 'Download images locally' }, - { name: 'wait', type: 'int', default: 3, help: 'Seconds to wait after page load' }, - { name: 'wait-for', valueRequired: true, help: 'CSS selector to wait for in the main document or same-origin iframes' }, - { name: 'wait-until', default: 'domstable', choices: ['domstable', 'networkidle'], help: 'Readiness policy after navigation: domstable or networkidle' }, - { name: 'frames', default: 'same-origin', choices: ['same-origin', 'all-same-origin', 'none'], help: 'Iframe handling mode: relevant same-origin, all-same-origin, or none' }, - { name: 'diagnose', type: 'boolean', default: false, help: 'Print render diagnostics (frames, empty containers, XHR/API-like requests) to stderr' }, - { name: 'stdout', type: 'boolean', default: false, help: 'Print markdown to stdout instead of saving to a file' }, - ], - columns: ['title', 'author', 'publish_time', 'status', 'size', 'saved'], - func: async (page, kwargs, debug) => (await import('./browser.js')).runFetchBrowser(page, kwargs, debug), -}); - export const webFetchCommand = cli({ site: 'web', name: 'fetch', access: 'read', strategy: Strategy.PUBLIC, browser: false, clientOwned: true, diff --git a/src/package-exports.test.ts b/src/package-exports.test.ts index 835957ea..d4afed18 100644 --- a/src/package-exports.test.ts +++ b/src/package-exports.test.ts @@ -9,6 +9,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import { fileURLToPath } from 'node:url'; import { buildManifest, buildManifestArtifacts } from './build-manifest.js'; +import { getRegistry } from './registry.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(__dirname, '..'); @@ -69,6 +70,23 @@ describe('adapter packaging', () => { .toBe('./dist/src/fetch/command.js'); }); + it('generates only web/fetch and its command package export', async () => { + const { entries } = await buildManifest(); + const webCommands = entries + .filter(entry => entry.site === 'web') + .map(entry => `${entry.site}/${entry.name}`); + const registeredWebCommands = [...getRegistry().values()] + .filter(command => command.site === 'web') + .map(command => `${command.site}/${command.name}`) + .sort(); + const fetchExports = Object.keys(pkgJson.exports as Record) + .filter(exportPath => exportPath.startsWith('./fetch/')); + + expect(webCommands).toEqual(['web/fetch']); + expect(registeredWebCommands).toEqual(['web/fetch']); + expect(fetchExports).toEqual(['./fetch/command']); + }); + it('publishes only client-owned web/fetch in generated artifacts', async () => { const { entries } = await buildManifest(); const artifacts = buildManifestArtifacts(entries, String(pkgJson.version), []); diff --git a/tests/e2e/article-download-pipeline.test.ts b/tests/e2e/article-download-pipeline.test.ts deleted file mode 100644 index 76a2e7c3..00000000 --- a/tests/e2e/article-download-pipeline.test.ts +++ /dev/null @@ -1,174 +0,0 @@ -/** - * E2E regression tests for the HTML → Markdown article pipeline. - * - * Drives real pages through `webcmd web fetch-browser` and asserts the hardened - * converter's invariants hold on the produced file: - * - no base64 `data:image/…` leaks - * - no