diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 6ed9c80..7a56472 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "2.7.0" + ".": "2.8.0" } diff --git a/.stats.yml b/.stats.yml index 33b7080..3b484e3 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ configured_endpoints: 37 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-960cb623c7ec84bf4dc0f5945cbc19eec9cca48271071f400d96066eaa55dbd6.yml -openapi_spec_hash: 84fd39e3f4dc964bf0c32d4e95da1b34 +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-758c3a2fbd5b7be61c8e6e0ad2e6a5ec30695747bb675960d888c497647d13d7.yml +openapi_spec_hash: 00002d90bde02f67e174368ae470c597 config_hash: 2bea1743c84d63bd61f8501a6ea63065 diff --git a/CHANGELOG.md b/CHANGELOG.md index 6251975..f345f52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## 2.8.0 (2026-08-01) + +Full Changelog: [v2.7.0...v2.8.0](https://github.com/context-dot-dev/context-typescript-sdk/compare/v2.7.0...v2.8.0) + +### Features + +* **api:** api update ([25fb630](https://github.com/context-dot-dev/context-typescript-sdk/commit/25fb6308fc3014add60d93b50cef3798faba6228)) +* **api:** api update ([085497a](https://github.com/context-dot-dev/context-typescript-sdk/commit/085497a10fec2c8751c2fb05d9874d250f5db21e)) +* **api:** api update ([a7943e0](https://github.com/context-dot-dev/context-typescript-sdk/commit/a7943e0755ae10cec275d64884d2cac98313049b)) + ## 2.7.0 (2026-07-31) Full Changelog: [v2.6.0...v2.7.0](https://github.com/context-dot-dev/context-typescript-sdk/compare/v2.6.0...v2.7.0) diff --git a/api.md b/api.md index 7e96fd9..083e0f0 100644 --- a/api.md +++ b/api.md @@ -123,8 +123,10 @@ Methods: Types: -- ErrorCount -- Error +- PageErrorCount +- Failure +- CrawlControls +- Intake - BatchRetrieveResponse - BatchListResponse - BatchCancelResponse diff --git a/package.json b/package.json index 2c4dd1a..46f2c6a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "context.dev", - "version": "2.7.0", + "version": "2.8.0", "description": "The official TypeScript library for the Context Dev API", "author": "Context Dev ", "types": "dist/index.d.ts", diff --git a/packages/mcp-server/manifest.json b/packages/mcp-server/manifest.json index 82f5835..6599567 100644 --- a/packages/mcp-server/manifest.json +++ b/packages/mcp-server/manifest.json @@ -1,7 +1,7 @@ { "dxt_version": "0.2", "name": "context.dev-mcp", - "version": "2.7.0", + "version": "2.8.0", "description": "The official MCP Server for the Context Dev API", "author": { "name": "Context Dev", diff --git a/packages/mcp-server/package.json b/packages/mcp-server/package.json index f80a63e..6acaeef 100644 --- a/packages/mcp-server/package.json +++ b/packages/mcp-server/package.json @@ -1,6 +1,6 @@ { "name": "context.dev-mcp", - "version": "2.7.0", + "version": "2.8.0", "description": "The official MCP Server for the Context Dev API", "author": "Context Dev ", "types": "dist/index.d.ts", diff --git a/packages/mcp-server/src/local-docs-search.ts b/packages/mcp-server/src/local-docs-search.ts index 1ff7a15..ab4050c 100644 --- a/packages/mcp-server/src/local-docs-search.ts +++ b/packages/mcp-server/src/local-docs-search.ts @@ -141,7 +141,7 @@ const EMBEDDED_METHODS: MethodEntry[] = [ response: "{ html: string; metadata: { finalUrl: string; sourceUrl: string; additionalMeta?: object; alternates?: { href: string; hreflang?: string; title?: string; type?: string; }[]; author?: string; canonicalUrl?: string; description?: string; favicon?: string; image?: string; jsonLd?: object[]; keywords?: string[]; language?: string; modifiedTime?: string; openGraph?: object; publishedTime?: string; robots?: string; siteName?: string; title?: string; twitter?: object; }; success: true; type: string; url: string; actionsApplied?: { instruction: string; status: 'applied' | 'failed' | 'skipped'; completionEvidence?: string; durationMs?: number; error?: string; method?: string; targetDescription?: string; }[]; actionsHtmlStale?: boolean; key_metadata?: { credits_consumed: number; credits_remaining: number; }; }", markdown: - "## web_scrape_html\n\n`client.web.webScrapeHTML(url: string, actions?: { do: 'wait'; timeMs: number; } | { action: string; do: 'perform'; }[], country?: string, excludeSelectors?: string[], headers?: object, includeFrames?: boolean | 'true' | 'false', includeSelectors?: string[], maxAgeMs?: number, pdf?: { end?: number; ocr?: boolean | 'true' | 'false'; shouldParse?: boolean | 'true' | 'false'; start?: number; }, settleAnimations?: boolean | 'true' | 'false', tags?: string[], timeoutMS?: number, useMainContentOnly?: boolean | 'true' | 'false', waitForMs?: number, zdr?: 'enabled' | 'disabled'): { html: string; metadata: object; success: true; type: string; url: string; actionsApplied?: object[]; actionsHtmlStale?: boolean; key_metadata?: object; }`\n\n**get** `/web/scrape/html`\n\nScrapes the given URL and returns the raw HTML content of the page. The base request costs 1 credit; requests with browser actions cost 2 credits.\n\n### Parameters\n\n- `url: string`\n Full URL to scrape (must include http:// or https:// protocol)\n\n- `actions?: { do: 'wait'; timeMs: number; } | { action: string; do: 'perform'; }[]`\n Optional browser actions executed in array order after the page loads and before content is captured. Requires a paid plan. Send a JSON array in the query parameter. Maximum: 5 actions.\n\n- `country?: string`\n Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev residential proxy exit location. Must be one of Context.dev's supported countries. When provided, Context.dev fetches the target page from that country.\n\n- `excludeSelectors?: string[]`\n CSS selectors to remove from the result. Applied after includeSelectors. Exclusion takes precedence: an element matching both is removed. Examples: \"nav\", \"footer\", \".ad-banner\", \"[aria-hidden=true]\".\n\n- `headers?: object`\n Optional outbound HTTP headers forwarded only to the target URL, sent as deep-object query params such as headers[X-Custom]=value. When provided, caching is bypassed: the result is neither read from nor written to cache.\n\n- `includeFrames?: boolean | 'true' | 'false'`\n When true, iframes are rendered inline into the returned HTML.\n\n- `includeSelectors?: string[]`\n CSS selectors. When provided, only matching subtrees (and their descendants) are kept and everything else is dropped. When omitted, the entire document is kept. Examples: \"article.main\", \"#content\", \"[role=main]\".\n\n- `maxAgeMs?: number`\n Return a cached result if a prior scrape for the same parameters exists and is younger than this many milliseconds. Defaults to 1 day (86400000 ms) when omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh.\n\n- `pdf?: { end?: number; ocr?: boolean | 'true' | 'false'; shouldParse?: boolean | 'true' | 'false'; start?: number; }`\n PDF parsing controls. Use start/end to limit text extraction and embedded-image detection/OCR to an inclusive 1-based page range.\n - `end?: number`\n Last 1-based PDF page to parse. When omitted, parsing ends at the last page. Must be greater than or equal to start when both are provided.\n - `ocr?: boolean | 'true' | 'false'`\n When true, detect and OCR images embedded in the selected PDF pages, inserting recognized text at each image's position in page reading order while preserving the PDF text layer. This is separate from automatic scanned-PDF OCR fallback.\n - `shouldParse?: boolean | 'true' | 'false'`\n When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and a 400 WEBSITE_ACCESS_ERROR is returned.\n - `start?: number`\n First 1-based PDF page to parse. When omitted, parsing starts at the first page.\n\n- `settleAnimations?: boolean | 'true' | 'false'`\n When true, waits briefly for CSS and transition animations to settle before extracting HTML. Defaults to false. This adds a bit of latency in exchange for more stable output on animated pages.\n\n- `tags?: string[]`\n Optional comma-separated caller-defined tags for tracking this request. Tags are recorded on the request's usage log and can be used to filter usage on the dashboard usage page. Up to 20 tags, each 1-50 characters.\n\n- `timeoutMS?: number`\n Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes).\n\n- `useMainContentOnly?: boolean | 'true' | 'false'`\n When true, return only the page's main content in the HTML response, excluding headers, footers, sidebars, and navigation when detectable.\n\n- `waitForMs?: number`\n Optional browser wait time in milliseconds after initial page load. Min: 0. Max: 30000 (30 seconds).\n\n- `zdr?: 'enabled' | 'disabled'`\n Set to enabled to bypass shared caches and omit request and response content from retained usage logs. Requires zero data retention to be enabled for your organization (contact support@context.dev), otherwise the request fails with ZDR_NOT_ENABLED. Successful ZDR responses include X-Context-ZDR: true.\n\n### Returns\n\n- `{ html: string; metadata: { finalUrl: string; sourceUrl: string; additionalMeta?: object; alternates?: { href: string; hreflang?: string; title?: string; type?: string; }[]; author?: string; canonicalUrl?: string; description?: string; favicon?: string; image?: string; jsonLd?: object[]; keywords?: string[]; language?: string; modifiedTime?: string; openGraph?: object; publishedTime?: string; robots?: string; siteName?: string; title?: string; twitter?: object; }; success: true; type: string; url: string; actionsApplied?: { instruction: string; status: 'applied' | 'failed' | 'skipped'; completionEvidence?: string; durationMs?: number; error?: string; method?: string; targetDescription?: string; }[]; actionsHtmlStale?: boolean; key_metadata?: { credits_consumed: number; credits_remaining: number; }; }`\n\n - `html: string`\n - `metadata: { finalUrl: string; sourceUrl: string; additionalMeta?: object; alternates?: { href: string; hreflang?: string; title?: string; type?: string; }[]; author?: string; canonicalUrl?: string; description?: string; favicon?: string; image?: string; jsonLd?: object[]; keywords?: string[]; language?: string; modifiedTime?: string; openGraph?: object; publishedTime?: string; robots?: string; siteName?: string; title?: string; twitter?: object; }`\n - `success: true`\n - `type: string`\n - `url: string`\n - `actionsApplied?: { instruction: string; status: 'applied' | 'failed' | 'skipped'; completionEvidence?: string; durationMs?: number; error?: string; method?: string; targetDescription?: string; }[]`\n - `actionsHtmlStale?: boolean`\n - `key_metadata?: { credits_consumed: number; credits_remaining: number; }`\n\n### Example\n\n```typescript\nimport ContextDev from 'context.dev';\n\nconst client = new ContextDev();\n\nconst response = await client.web.webScrapeHTML({ url: 'https://example.com' });\n\nconsole.log(response);\n```", + "## web_scrape_html\n\n`client.web.webScrapeHTML(url: string, actions?: { do: 'wait'; timeMs: number; } | { action: string; do: 'perform'; }[], country?: string, excludeSelectors?: string[], headers?: object, includeFrames?: boolean | 'true' | 'false', includeSelectors?: string[], maxAgeMs?: number, pdf?: { end?: number; ocr?: boolean | 'true' | 'false'; shouldParse?: boolean | 'true' | 'false'; start?: number; }, settleAnimations?: boolean | 'true' | 'false', tags?: string[], timeoutMS?: number, useMainContentOnly?: boolean | 'true' | 'false', waitForMs?: number, zdr?: 'enabled' | 'disabled'): { html: string; metadata: object; success: true; type: string; url: string; actionsApplied?: object[]; actionsHtmlStale?: boolean; key_metadata?: object; }`\n\n**get** `/web/scrape/html`\n\nScrapes the given URL and returns the raw HTML content of the page. The base request costs 1 credit; requests with browser actions cost 2 credits.\n\n### Parameters\n\n- `url: string`\n Full URL to scrape (must include http:// or https:// protocol)\n\n- `actions?: { do: 'wait'; timeMs: number; } | { action: string; do: 'perform'; }[]`\n Optional browser actions executed in array order after the page loads and before content is captured. Requires a paid plan. Send a JSON array in the query parameter. Maximum: 5 actions.\n\n- `country?: string`\n Fetch the target page through a residential proxy in this country (ISO 3166-1 alpha-2).\n\n- `excludeSelectors?: string[]`\n CSS selectors to remove from the result. Applied after includeSelectors. Exclusion takes precedence: an element matching both is removed. Examples: \"nav\", \"footer\", \".ad-banner\", \"[aria-hidden=true]\".\n\n- `headers?: object`\n Optional outbound HTTP headers forwarded only to the target URL, sent as deep-object query params such as headers[X-Custom]=value. When provided, caching is bypassed: the result is neither read from nor written to cache.\n\n- `includeFrames?: boolean | 'true' | 'false'`\n When true, iframes are rendered inline into the returned HTML.\n\n- `includeSelectors?: string[]`\n CSS selectors. When provided, only matching subtrees (and their descendants) are kept and everything else is dropped. When omitted, the entire document is kept. Examples: \"article.main\", \"#content\", \"[role=main]\".\n\n- `maxAgeMs?: number`\n Return a cached result if a prior scrape for the same parameters exists and is younger than this many milliseconds. Defaults to 1 day (86400000 ms) when omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh.\n\n- `pdf?: { end?: number; ocr?: boolean | 'true' | 'false'; shouldParse?: boolean | 'true' | 'false'; start?: number; }`\n PDF parsing controls. Use start/end to limit text extraction and embedded-image detection/OCR to an inclusive 1-based page range.\n - `end?: number`\n Last 1-based PDF page to parse. When omitted, parsing ends at the last page. Must be greater than or equal to start when both are provided.\n - `ocr?: boolean | 'true' | 'false'`\n When true, detect and OCR images embedded in the selected PDF pages, inserting recognized text at each image's position in page reading order while preserving the PDF text layer. This is separate from automatic scanned-PDF OCR fallback.\n - `shouldParse?: boolean | 'true' | 'false'`\n When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and a 400 WEBSITE_ACCESS_ERROR is returned.\n - `start?: number`\n First 1-based PDF page to parse. When omitted, parsing starts at the first page.\n\n- `settleAnimations?: boolean | 'true' | 'false'`\n When true, waits briefly for CSS and transition animations to settle before extracting HTML. Defaults to false. This adds a bit of latency in exchange for more stable output on animated pages.\n\n- `tags?: string[]`\n Optional comma-separated caller-defined tags for tracking this request. Tags are recorded on the request's usage log and can be used to filter usage on the dashboard usage page. Up to 20 tags, each 1-50 characters.\n\n- `timeoutMS?: number`\n Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes).\n\n- `useMainContentOnly?: boolean | 'true' | 'false'`\n When true, return only the page's main content in the HTML response, excluding headers, footers, sidebars, and navigation when detectable.\n\n- `waitForMs?: number`\n Optional browser wait time in milliseconds after initial page load. Min: 0. Max: 30000 (30 seconds).\n\n- `zdr?: 'enabled' | 'disabled'`\n Set to enabled to bypass shared caches and omit request and response content from retained usage logs. Requires zero data retention to be enabled for your organization (contact support@context.dev), otherwise the request fails with ZDR_NOT_ENABLED. Successful ZDR responses include X-Context-ZDR: true.\n\n### Returns\n\n- `{ html: string; metadata: { finalUrl: string; sourceUrl: string; additionalMeta?: object; alternates?: { href: string; hreflang?: string; title?: string; type?: string; }[]; author?: string; canonicalUrl?: string; description?: string; favicon?: string; image?: string; jsonLd?: object[]; keywords?: string[]; language?: string; modifiedTime?: string; openGraph?: object; publishedTime?: string; robots?: string; siteName?: string; title?: string; twitter?: object; }; success: true; type: string; url: string; actionsApplied?: { instruction: string; status: 'applied' | 'failed' | 'skipped'; completionEvidence?: string; durationMs?: number; error?: string; method?: string; targetDescription?: string; }[]; actionsHtmlStale?: boolean; key_metadata?: { credits_consumed: number; credits_remaining: number; }; }`\n\n - `html: string`\n - `metadata: { finalUrl: string; sourceUrl: string; additionalMeta?: object; alternates?: { href: string; hreflang?: string; title?: string; type?: string; }[]; author?: string; canonicalUrl?: string; description?: string; favicon?: string; image?: string; jsonLd?: object[]; keywords?: string[]; language?: string; modifiedTime?: string; openGraph?: object; publishedTime?: string; robots?: string; siteName?: string; title?: string; twitter?: object; }`\n - `success: true`\n - `type: string`\n - `url: string`\n - `actionsApplied?: { instruction: string; status: 'applied' | 'failed' | 'skipped'; completionEvidence?: string; durationMs?: number; error?: string; method?: string; targetDescription?: string; }[]`\n - `actionsHtmlStale?: boolean`\n - `key_metadata?: { credits_consumed: number; credits_remaining: number; }`\n\n### Example\n\n```typescript\nimport ContextDev from 'context.dev';\n\nconst client = new ContextDev();\n\nconst response = await client.web.webScrapeHTML({ url: 'https://example.com' });\n\nconsole.log(response);\n```", perLanguage: { typescript: { method: 'client.web.webScrapeHTML', @@ -211,7 +211,7 @@ const EMBEDDED_METHODS: MethodEntry[] = [ response: "{ contentLength: number; markdown: string; metadata: { finalUrl: string; sourceUrl: string; additionalMeta?: object; alternates?: { href: string; hreflang?: string; title?: string; type?: string; }[]; author?: string; canonicalUrl?: string; description?: string; favicon?: string; image?: string; jsonLd?: object[]; keywords?: string[]; language?: string; modifiedTime?: string; openGraph?: object; publishedTime?: string; robots?: string; siteName?: string; title?: string; twitter?: object; }; success: true; url: string; actionsApplied?: { instruction: string; status: 'applied' | 'failed' | 'skipped'; completionEvidence?: string; durationMs?: number; error?: string; method?: string; targetDescription?: string; }[]; actionsHtmlStale?: boolean; key_metadata?: { credits_consumed: number; credits_remaining: number; }; }", markdown: - "## web_scrape_md\n\n`client.web.webScrapeMd(url: string, actions?: { do: 'wait'; timeMs: number; } | { action: string; do: 'perform'; }[], country?: string, excludeSelectors?: string[], headers?: object, includeFrames?: boolean | 'true' | 'false', includeImages?: boolean | 'true' | 'false', includeLinks?: boolean | 'true' | 'false', includeSelectors?: string[], maxAgeMs?: number, pdf?: { end?: number; ocr?: boolean | 'true' | 'false'; shouldParse?: boolean | 'true' | 'false'; start?: number; }, settleAnimations?: boolean | 'true' | 'false', shortenBase64Images?: boolean | 'true' | 'false', tags?: string[], timeoutMS?: number, useMainContentOnly?: boolean | 'true' | 'false', waitForMs?: number, zdr?: 'enabled' | 'disabled'): { contentLength: number; markdown: string; metadata: object; success: true; url: string; actionsApplied?: object[]; actionsHtmlStale?: boolean; key_metadata?: object; }`\n\n**get** `/web/scrape/markdown`\n\nScrapes the given URL into LLM usable Markdown. Inspect key_metadata on JSON responses from a recognized API key; use error_code to distinguish stable failure categories.\n\n### Billing & errors\n\n| HTTP status | Billed? | Meaning |\n| --- | --- | --- |\n| 200 | Yes — 1 credit, or 2 credits with actions | Successful scrape, including a zero-length result when includeSelectors matched nothing |\n| 400 | No | Invalid input, skipped PDF, or the page could not be scraped |\n| 401 / 403 | No | Invalid/disabled key, insufficient permissions, or credits exhausted; inspect error_code |\n| 404 | No | Target page returned or fingerprinted as not found |\n| 408 | No | Request timed out |\n| 415 | No | Unsupported content type |\n| 429 | No | Per-minute rate limit exceeded; honor Retry-After |\n| 500 | No | Internal error |\n\n### Parameters\n\n- `url: string`\n Full URL to scrape into LLM usable Markdown (must include http:// or https:// protocol)\n\n- `actions?: { do: 'wait'; timeMs: number; } | { action: string; do: 'perform'; }[]`\n Optional browser actions executed in array order after the page loads and before content is captured. Requires a paid plan. Send a JSON array in the query parameter. Maximum: 5 actions.\n\n- `country?: string`\n Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev residential proxy exit location. Must be one of Context.dev's supported countries. When provided, Context.dev fetches the target page from that country.\n\n- `excludeSelectors?: string[]`\n CSS selectors to remove before conversion to Markdown. Applied after includeSelectors. Exclusion takes precedence: an element matching both is removed. Examples: \"nav\", \"footer\", \".ad-banner\", \"[aria-hidden=true]\".\n\n- `headers?: object`\n Optional outbound HTTP headers forwarded only to the target URL, sent as deep-object query params such as headers[X-Custom]=value. When provided, caching is bypassed: the result is neither read from nor written to cache.\n\n- `includeFrames?: boolean | 'true' | 'false'`\n When true, the contents of iframes are rendered to Markdown.\n\n- `includeImages?: boolean | 'true' | 'false'`\n Include image references in Markdown output\n\n- `includeLinks?: boolean | 'true' | 'false'`\n Preserve hyperlinks in Markdown output\n\n- `includeSelectors?: string[]`\n CSS selectors. When provided, only matching HTML subtrees (and their descendants) are kept before conversion to Markdown. When omitted, the entire document is kept. Examples: \"article.main\", \"#content\", \"[role=main]\".\n\n- `maxAgeMs?: number`\n Return a cached result if a prior scrape for the same parameters exists and is younger than this many milliseconds. Defaults to 1 day (86400000 ms) when omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh.\n\n- `pdf?: { end?: number; ocr?: boolean | 'true' | 'false'; shouldParse?: boolean | 'true' | 'false'; start?: number; }`\n PDF parsing controls. Use start/end to limit text extraction and embedded-image detection/OCR to an inclusive 1-based page range.\n - `end?: number`\n Last 1-based PDF page to parse. When omitted, parsing ends at the last page. Must be greater than or equal to start when both are provided.\n - `ocr?: boolean | 'true' | 'false'`\n When true, detect and OCR images embedded in the selected PDF pages, inserting recognized text at each image's position in page reading order while preserving the PDF text layer. This is separate from automatic scanned-PDF OCR fallback.\n - `shouldParse?: boolean | 'true' | 'false'`\n When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and a 400 WEBSITE_ACCESS_ERROR is returned.\n - `start?: number`\n First 1-based PDF page to parse. When omitted, parsing starts at the first page.\n\n- `settleAnimations?: boolean | 'true' | 'false'`\n When true, waits briefly for CSS and transition animations to settle before converting to Markdown. Defaults to false. This adds a bit of latency in exchange for more stable output on animated pages.\n\n- `shortenBase64Images?: boolean | 'true' | 'false'`\n Shorten base64-encoded image data in the Markdown output\n\n- `tags?: string[]`\n Optional comma-separated caller-defined tags for tracking this request. Tags are recorded on the request's usage log and can be used to filter usage on the dashboard usage page. Up to 20 tags, each 1-50 characters.\n\n- `timeoutMS?: number`\n Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes).\n\n- `useMainContentOnly?: boolean | 'true' | 'false'`\n Extract only the main content of the page, excluding headers, footers, sidebars, and navigation\n\n- `waitForMs?: number`\n Optional browser wait time in milliseconds after initial page load before converting the page to Markdown. Min: 0. Max: 30000 (30 seconds).\n\n- `zdr?: 'enabled' | 'disabled'`\n Set to enabled to bypass shared caches and omit request and response content from retained usage logs. Requires zero data retention to be enabled for your organization (contact support@context.dev), otherwise the request fails with ZDR_NOT_ENABLED. Successful ZDR responses include X-Context-ZDR: true.\n\n### Returns\n\n- `{ contentLength: number; markdown: string; metadata: { finalUrl: string; sourceUrl: string; additionalMeta?: object; alternates?: { href: string; hreflang?: string; title?: string; type?: string; }[]; author?: string; canonicalUrl?: string; description?: string; favicon?: string; image?: string; jsonLd?: object[]; keywords?: string[]; language?: string; modifiedTime?: string; openGraph?: object; publishedTime?: string; robots?: string; siteName?: string; title?: string; twitter?: object; }; success: true; url: string; actionsApplied?: { instruction: string; status: 'applied' | 'failed' | 'skipped'; completionEvidence?: string; durationMs?: number; error?: string; method?: string; targetDescription?: string; }[]; actionsHtmlStale?: boolean; key_metadata?: { credits_consumed: number; credits_remaining: number; }; }`\n\n - `contentLength: number`\n - `markdown: string`\n - `metadata: { finalUrl: string; sourceUrl: string; additionalMeta?: object; alternates?: { href: string; hreflang?: string; title?: string; type?: string; }[]; author?: string; canonicalUrl?: string; description?: string; favicon?: string; image?: string; jsonLd?: object[]; keywords?: string[]; language?: string; modifiedTime?: string; openGraph?: object; publishedTime?: string; robots?: string; siteName?: string; title?: string; twitter?: object; }`\n - `success: true`\n - `url: string`\n - `actionsApplied?: { instruction: string; status: 'applied' | 'failed' | 'skipped'; completionEvidence?: string; durationMs?: number; error?: string; method?: string; targetDescription?: string; }[]`\n - `actionsHtmlStale?: boolean`\n - `key_metadata?: { credits_consumed: number; credits_remaining: number; }`\n\n### Example\n\n```typescript\nimport ContextDev from 'context.dev';\n\nconst client = new ContextDev();\n\nconst response = await client.web.webScrapeMd({ url: 'https://example.com' });\n\nconsole.log(response);\n```", + "## web_scrape_md\n\n`client.web.webScrapeMd(url: string, actions?: { do: 'wait'; timeMs: number; } | { action: string; do: 'perform'; }[], country?: string, excludeSelectors?: string[], headers?: object, includeFrames?: boolean | 'true' | 'false', includeImages?: boolean | 'true' | 'false', includeLinks?: boolean | 'true' | 'false', includeSelectors?: string[], maxAgeMs?: number, pdf?: { end?: number; ocr?: boolean | 'true' | 'false'; shouldParse?: boolean | 'true' | 'false'; start?: number; }, settleAnimations?: boolean | 'true' | 'false', shortenBase64Images?: boolean | 'true' | 'false', tags?: string[], timeoutMS?: number, useMainContentOnly?: boolean | 'true' | 'false', waitForMs?: number, zdr?: 'enabled' | 'disabled'): { contentLength: number; markdown: string; metadata: object; success: true; url: string; actionsApplied?: object[]; actionsHtmlStale?: boolean; key_metadata?: object; }`\n\n**get** `/web/scrape/markdown`\n\nScrapes the given URL into LLM usable Markdown. Inspect key_metadata on JSON responses from a recognized API key; use error_code to distinguish stable failure categories.\n\n### Billing & errors\n\n| HTTP status | Billed? | Meaning |\n| --- | --- | --- |\n| 200 | Yes — 1 credit, or 2 credits with actions | Successful scrape, including a zero-length result when includeSelectors matched nothing |\n| 400 | No | Invalid input, skipped PDF, or the page could not be scraped |\n| 401 / 403 | No | Invalid/disabled key, insufficient permissions, or credits exhausted; inspect error_code |\n| 404 | No | Target page returned or fingerprinted as not found |\n| 408 | No | Request timed out |\n| 415 | No | Unsupported content type |\n| 429 | No | Per-minute rate limit exceeded; honor Retry-After |\n| 500 | No | Internal error |\n\n### Parameters\n\n- `url: string`\n Full URL to scrape into LLM usable Markdown (must include http:// or https:// protocol)\n\n- `actions?: { do: 'wait'; timeMs: number; } | { action: string; do: 'perform'; }[]`\n Optional browser actions executed in array order after the page loads and before content is captured. Requires a paid plan. Send a JSON array in the query parameter. Maximum: 5 actions.\n\n- `country?: string`\n Fetch the target page through a residential proxy in this country (ISO 3166-1 alpha-2).\n\n- `excludeSelectors?: string[]`\n CSS selectors to remove before conversion to Markdown. Applied after includeSelectors. Exclusion takes precedence: an element matching both is removed. Examples: \"nav\", \"footer\", \".ad-banner\", \"[aria-hidden=true]\".\n\n- `headers?: object`\n Optional outbound HTTP headers forwarded only to the target URL, sent as deep-object query params such as headers[X-Custom]=value. When provided, caching is bypassed: the result is neither read from nor written to cache.\n\n- `includeFrames?: boolean | 'true' | 'false'`\n When true, the contents of iframes are rendered to Markdown.\n\n- `includeImages?: boolean | 'true' | 'false'`\n Include image references in Markdown output\n\n- `includeLinks?: boolean | 'true' | 'false'`\n Preserve hyperlinks in Markdown output\n\n- `includeSelectors?: string[]`\n CSS selectors. When provided, only matching HTML subtrees (and their descendants) are kept before conversion to Markdown. When omitted, the entire document is kept. Examples: \"article.main\", \"#content\", \"[role=main]\".\n\n- `maxAgeMs?: number`\n Return a cached result if a prior scrape for the same parameters exists and is younger than this many milliseconds. Defaults to 1 day (86400000 ms) when omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh.\n\n- `pdf?: { end?: number; ocr?: boolean | 'true' | 'false'; shouldParse?: boolean | 'true' | 'false'; start?: number; }`\n PDF parsing controls. Use start/end to limit text extraction and embedded-image detection/OCR to an inclusive 1-based page range.\n - `end?: number`\n Last 1-based PDF page to parse. When omitted, parsing ends at the last page. Must be greater than or equal to start when both are provided.\n - `ocr?: boolean | 'true' | 'false'`\n When true, detect and OCR images embedded in the selected PDF pages, inserting recognized text at each image's position in page reading order while preserving the PDF text layer. This is separate from automatic scanned-PDF OCR fallback.\n - `shouldParse?: boolean | 'true' | 'false'`\n When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and a 400 WEBSITE_ACCESS_ERROR is returned.\n - `start?: number`\n First 1-based PDF page to parse. When omitted, parsing starts at the first page.\n\n- `settleAnimations?: boolean | 'true' | 'false'`\n When true, waits briefly for CSS and transition animations to settle before converting to Markdown. Defaults to false. This adds a bit of latency in exchange for more stable output on animated pages.\n\n- `shortenBase64Images?: boolean | 'true' | 'false'`\n Shorten base64-encoded image data in the Markdown output\n\n- `tags?: string[]`\n Optional comma-separated caller-defined tags for tracking this request. Tags are recorded on the request's usage log and can be used to filter usage on the dashboard usage page. Up to 20 tags, each 1-50 characters.\n\n- `timeoutMS?: number`\n Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes).\n\n- `useMainContentOnly?: boolean | 'true' | 'false'`\n Extract only the main content of the page, excluding headers, footers, sidebars, and navigation\n\n- `waitForMs?: number`\n Optional browser wait time in milliseconds after initial page load before converting the page to Markdown. Min: 0. Max: 30000 (30 seconds).\n\n- `zdr?: 'enabled' | 'disabled'`\n Set to enabled to bypass shared caches and omit request and response content from retained usage logs. Requires zero data retention to be enabled for your organization (contact support@context.dev), otherwise the request fails with ZDR_NOT_ENABLED. Successful ZDR responses include X-Context-ZDR: true.\n\n### Returns\n\n- `{ contentLength: number; markdown: string; metadata: { finalUrl: string; sourceUrl: string; additionalMeta?: object; alternates?: { href: string; hreflang?: string; title?: string; type?: string; }[]; author?: string; canonicalUrl?: string; description?: string; favicon?: string; image?: string; jsonLd?: object[]; keywords?: string[]; language?: string; modifiedTime?: string; openGraph?: object; publishedTime?: string; robots?: string; siteName?: string; title?: string; twitter?: object; }; success: true; url: string; actionsApplied?: { instruction: string; status: 'applied' | 'failed' | 'skipped'; completionEvidence?: string; durationMs?: number; error?: string; method?: string; targetDescription?: string; }[]; actionsHtmlStale?: boolean; key_metadata?: { credits_consumed: number; credits_remaining: number; }; }`\n\n - `contentLength: number`\n - `markdown: string`\n - `metadata: { finalUrl: string; sourceUrl: string; additionalMeta?: object; alternates?: { href: string; hreflang?: string; title?: string; type?: string; }[]; author?: string; canonicalUrl?: string; description?: string; favicon?: string; image?: string; jsonLd?: object[]; keywords?: string[]; language?: string; modifiedTime?: string; openGraph?: object; publishedTime?: string; robots?: string; siteName?: string; title?: string; twitter?: object; }`\n - `success: true`\n - `url: string`\n - `actionsApplied?: { instruction: string; status: 'applied' | 'failed' | 'skipped'; completionEvidence?: string; durationMs?: number; error?: string; method?: string; targetDescription?: string; }[]`\n - `actionsHtmlStale?: boolean`\n - `key_metadata?: { credits_consumed: number; credits_remaining: number; }`\n\n### Example\n\n```typescript\nimport ContextDev from 'context.dev';\n\nconst client = new ContextDev();\n\nconst response = await client.web.webScrapeMd({ url: 'https://example.com' });\n\nconsole.log(response);\n```", perLanguage: { typescript: { method: 'client.web.webScrapeMd', @@ -394,7 +394,7 @@ const EMBEDDED_METHODS: MethodEntry[] = [ response: "{ code?: number; domain?: string; height?: number; key_metadata?: { credits_consumed: number; credits_remaining: number; }; screenshot?: string; screenshotType?: 'viewport' | 'fullPage'; status?: string; width?: number; }", markdown: - "## screenshot\n\n`client.web.screenshot(colorScheme?: 'light' | 'dark', country?: string, directUrl?: string, domain?: string, fullScreenshot?: 'true' | 'false', handleCookiePopup?: boolean | 'true' | 'false', maxAgeMs?: number, page?: 'login' | 'signup' | 'blog' | 'careers' | 'pricing' | 'terms' | 'privacy' | 'contact', scrollOffset?: number, tags?: string[], timeoutMS?: number, viewport?: { height?: number; width?: number; }, waitForMs?: number, zdr?: 'enabled' | 'disabled'): { code?: number; domain?: string; height?: number; key_metadata?: object; screenshot?: string; screenshotType?: 'viewport' | 'fullPage'; status?: string; width?: number; }`\n\n**get** `/web/screenshot`\n\nCapture a screenshot of a website.\n\n### Parameters\n\n- `colorScheme?: 'light' | 'dark'`\n Optional parameter to choose the site's visual theme in the screenshot. Use 'light' or 'dark' when the site offers both appearances.\n\n- `country?: string`\n Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev residential proxy exit location. Must be one of Context.dev's supported countries. When provided, Context.dev fetches the target page from that country.\n\n- `directUrl?: string`\n A specific URL to screenshot directly, bypassing domain resolution (e.g., 'https://example.com/pricing'). When provided, the screenshot is taken of this exact URL. You must provide either 'domain' or 'directUrl', but not both.\n\n- `domain?: string`\n Domain name to take screenshot of (e.g., 'example.com', 'google.com'). The domain will be automatically normalized and validated. You must provide either 'domain' or 'directUrl', but not both.\n\n- `fullScreenshot?: 'true' | 'false'`\n Optional parameter to determine screenshot type. If 'true', takes a full page screenshot capturing all content. If 'false' or not provided, takes a viewport screenshot (standard browser view).\n\n- `handleCookiePopup?: boolean | 'true' | 'false'`\n Optional parameter to control cookie/consent popup handling. If 'true', we dismiss cookie banner before capture. If 'false' or not provided, captures the page without that step.\n\n- `maxAgeMs?: number`\n Return a cached screenshot if a prior screenshot for the same parameters exists and is younger than this many milliseconds. Defaults to 1 day (86400000 ms) when omitted. Max is 30 days (2592000000 ms). Set to 0 to always capture fresh.\n\n- `page?: 'login' | 'signup' | 'blog' | 'careers' | 'pricing' | 'terms' | 'privacy' | 'contact'`\n Optional parameter to specify which page type to screenshot. If provided, the system will scrape the domain's links and use heuristics to find the most appropriate URL for the specified page type (30 supported languages). If not provided, screenshots the main domain landing page. Only applicable when using 'domain', not 'directUrl'.\n\n- `scrollOffset?: number`\n Optional vertical scroll offset in pixels for capturing a long page in viewport-sized chunks. When provided, the full page is captured once and the returned image is the viewport-sized slice that begins at this Y offset (e.g. request scrollOffset=0, then 1080, then 2160 to walk a 1920x1080 landing page top to bottom). The final slice may be shorter than the viewport height. Takes precedence over fullScreenshot. Max: 100000.\n\n- `tags?: string[]`\n Optional comma-separated caller-defined tags for tracking this request. Tags are recorded on the request's usage log and can be used to filter usage on the dashboard usage page. Up to 20 tags, each 1-50 characters.\n\n- `timeoutMS?: number`\n Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes).\n\n- `viewport?: { height?: number; width?: number; }`\n Optional browser viewport dimensions for the screenshot. Defaults to 1920x1080.\n - `height?: number`\n Viewport height in pixels.\n - `width?: number`\n Viewport width in pixels.\n\n- `waitForMs?: number`\n Optional browser wait time in milliseconds after initial page load before taking the screenshot. Min: 0. Max: 30000 (30 seconds). Defaults to 3000 ms when omitted.\n\n- `zdr?: 'enabled' | 'disabled'`\n Set to enabled to bypass shared caches and omit request and response content from retained usage logs. Requires zero data retention to be enabled for your organization (contact support@context.dev), otherwise the request fails with ZDR_NOT_ENABLED. Successful ZDR responses include X-Context-ZDR: true.\n\n### Returns\n\n- `{ code?: number; domain?: string; height?: number; key_metadata?: { credits_consumed: number; credits_remaining: number; }; screenshot?: string; screenshotType?: 'viewport' | 'fullPage'; status?: string; width?: number; }`\n\n - `code?: number`\n - `domain?: string`\n - `height?: number`\n - `key_metadata?: { credits_consumed: number; credits_remaining: number; }`\n - `screenshot?: string`\n - `screenshotType?: 'viewport' | 'fullPage'`\n - `status?: string`\n - `width?: number`\n\n### Example\n\n```typescript\nimport ContextDev from 'context.dev';\n\nconst client = new ContextDev();\n\nconst response = await client.web.screenshot();\n\nconsole.log(response);\n```", + "## screenshot\n\n`client.web.screenshot(colorScheme?: 'light' | 'dark', country?: string, directUrl?: string, domain?: string, fullScreenshot?: 'true' | 'false', handleCookiePopup?: boolean | 'true' | 'false', maxAgeMs?: number, page?: 'login' | 'signup' | 'blog' | 'careers' | 'pricing' | 'terms' | 'privacy' | 'contact', scrollOffset?: number, tags?: string[], timeoutMS?: number, viewport?: { height?: number; width?: number; }, waitForMs?: number, zdr?: 'enabled' | 'disabled'): { code?: number; domain?: string; height?: number; key_metadata?: object; screenshot?: string; screenshotType?: 'viewport' | 'fullPage'; status?: string; width?: number; }`\n\n**get** `/web/screenshot`\n\nCapture a screenshot of a website.\n\n### Parameters\n\n- `colorScheme?: 'light' | 'dark'`\n Optional parameter to choose the site's visual theme in the screenshot. Use 'light' or 'dark' when the site offers both appearances.\n\n- `country?: string`\n Fetch the target page through a residential proxy in this country (ISO 3166-1 alpha-2).\n\n- `directUrl?: string`\n A specific URL to screenshot directly, bypassing domain resolution (e.g., 'https://example.com/pricing'). When provided, the screenshot is taken of this exact URL. You must provide either 'domain' or 'directUrl', but not both.\n\n- `domain?: string`\n Domain name to take screenshot of (e.g., 'example.com', 'google.com'). The domain will be automatically normalized and validated. You must provide either 'domain' or 'directUrl', but not both.\n\n- `fullScreenshot?: 'true' | 'false'`\n Optional parameter to determine screenshot type. If 'true', takes a full page screenshot capturing all content. If 'false' or not provided, takes a viewport screenshot (standard browser view).\n\n- `handleCookiePopup?: boolean | 'true' | 'false'`\n Optional parameter to control cookie/consent popup handling. If 'true', we dismiss cookie banner before capture. If 'false' or not provided, captures the page without that step.\n\n- `maxAgeMs?: number`\n Return a cached screenshot if a prior screenshot for the same parameters exists and is younger than this many milliseconds. Defaults to 1 day (86400000 ms) when omitted. Max is 30 days (2592000000 ms). Set to 0 to always capture fresh.\n\n- `page?: 'login' | 'signup' | 'blog' | 'careers' | 'pricing' | 'terms' | 'privacy' | 'contact'`\n Optional parameter to specify which page type to screenshot. If provided, the system will scrape the domain's links and use heuristics to find the most appropriate URL for the specified page type (30 supported languages). If not provided, screenshots the main domain landing page. Only applicable when using 'domain', not 'directUrl'.\n\n- `scrollOffset?: number`\n Optional vertical scroll offset in pixels for capturing a long page in viewport-sized chunks. When provided, the full page is captured once and the returned image is the viewport-sized slice that begins at this Y offset (e.g. request scrollOffset=0, then 1080, then 2160 to walk a 1920x1080 landing page top to bottom). The final slice may be shorter than the viewport height. Takes precedence over fullScreenshot. Max: 100000.\n\n- `tags?: string[]`\n Optional comma-separated caller-defined tags for tracking this request. Tags are recorded on the request's usage log and can be used to filter usage on the dashboard usage page. Up to 20 tags, each 1-50 characters.\n\n- `timeoutMS?: number`\n Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes).\n\n- `viewport?: { height?: number; width?: number; }`\n Optional browser viewport dimensions for the screenshot. Defaults to 1920x1080.\n - `height?: number`\n Viewport height in pixels.\n - `width?: number`\n Viewport width in pixels.\n\n- `waitForMs?: number`\n Optional browser wait time in milliseconds after initial page load before taking the screenshot. Min: 0. Max: 30000 (30 seconds). Defaults to 3000 ms when omitted.\n\n- `zdr?: 'enabled' | 'disabled'`\n Set to enabled to bypass shared caches and omit request and response content from retained usage logs. Requires zero data retention to be enabled for your organization (contact support@context.dev), otherwise the request fails with ZDR_NOT_ENABLED. Successful ZDR responses include X-Context-ZDR: true.\n\n### Returns\n\n- `{ code?: number; domain?: string; height?: number; key_metadata?: { credits_consumed: number; credits_remaining: number; }; screenshot?: string; screenshotType?: 'viewport' | 'fullPage'; status?: string; width?: number; }`\n\n - `code?: number`\n - `domain?: string`\n - `height?: number`\n - `key_metadata?: { credits_consumed: number; credits_remaining: number; }`\n - `screenshot?: string`\n - `screenshotType?: 'viewport' | 'fullPage'`\n - `status?: string`\n - `width?: number`\n\n### Example\n\n```typescript\nimport ContextDev from 'context.dev';\n\nconst client = new ContextDev();\n\nconst response = await client.web.screenshot();\n\nconsole.log(response);\n```", perLanguage: { typescript: { method: 'client.web.screenshot', @@ -466,7 +466,7 @@ const EMBEDDED_METHODS: MethodEntry[] = [ response: '{ metadata: { maxCrawlDepth: number; numFailed: number; numSkipped: number; numSucceeded: number; numUrls: number; }; results: { markdown: string; metadata: { crawlDepth: number; finalUrl: string; sourceUrl: string; statusCode: number; success: boolean; title: string; url: string; additionalMeta?: object; alternates?: object[]; author?: string; canonicalUrl?: string; description?: string; favicon?: string; image?: string; jsonLd?: object[]; keywords?: string[]; language?: string; modifiedTime?: string; openGraph?: object; publishedTime?: string; robots?: string; siteName?: string; twitter?: object; }; }[]; key_metadata?: { credits_consumed: number; credits_remaining: number; }; }', markdown: - "## web_crawl_md\n\n`client.web.webCrawlMd(url: string, country?: string, excludeSelectors?: string[], followSubdomains?: boolean, includeFrames?: boolean, includeImages?: boolean, includeLinks?: boolean, includeSelectors?: string[], maxAgeMs?: number, maxDepth?: number, maxPages?: number, pdf?: { end?: number; ocr?: boolean; shouldParse?: boolean; start?: number; }, settleAnimations?: boolean, shortenBase64Images?: boolean, stopAfterMs?: number, tags?: string[], timeoutMS?: number, urlRegex?: string, useMainContentOnly?: boolean, waitForMs?: number, zdr?: 'enabled' | 'disabled'): { metadata: object; results: object[]; key_metadata?: object; }`\n\n**post** `/web/crawl`\n\nPerforms a crawl starting from a given URL, extracts page content as Markdown, and returns results for all crawled pages.\n\n### Parameters\n\n- `url: string`\n The starting URL for the crawl (must include http:// or https:// protocol)\n\n- `country?: string`\n Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev residential proxy exit location. Must be one of Context.dev's supported countries. When provided, Context.dev fetches the target page from that country.\n\n- `excludeSelectors?: string[]`\n CSS selectors to remove before each crawled page is converted to Markdown. Applied after includeSelectors. Exclusion takes precedence: an element matching both is removed. Examples: \"nav\", \"footer\", \".ad-banner\", \"[aria-hidden=true]\".\n\n- `followSubdomains?: boolean`\n When true, follow links on subdomains of the starting URL's domain (e.g. docs.example.com when starting from example.com). www and apex are always treated as equivalent.\n\n- `includeFrames?: boolean`\n When true, the contents of iframes are rendered to Markdown for each crawled page.\n\n- `includeImages?: boolean`\n Include image references in the Markdown output\n\n- `includeLinks?: boolean`\n Preserve hyperlinks in the Markdown output\n\n- `includeSelectors?: string[]`\n CSS selectors. When provided, only matching HTML subtrees (and their descendants) are kept before each crawled page is converted to Markdown. When omitted, the entire document is kept. Examples: \"article.main\", \"#content\", \"[role=main]\".\n\n- `maxAgeMs?: number`\n Return a cached result if a prior scrape for the same parameters exists and is younger than this many milliseconds. Defaults to 1 day (86400000 ms) when omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh.\n\n- `maxDepth?: number`\n Maximum link depth from the starting URL (0 = only the starting page)\n\n- `maxPages?: number`\n Maximum number of pages to crawl. Hard cap: 500.\n\n- `pdf?: { end?: number; ocr?: boolean; shouldParse?: boolean; start?: number; }`\n PDF parsing controls. Use start/end to limit text extraction and embedded-image detection/OCR to an inclusive 1-based page range.\n - `end?: number`\n Last 1-based PDF page to parse. When omitted, parsing ends at the last page. Must be greater than or equal to start when both are provided.\n - `ocr?: boolean`\n When true, detect and OCR images embedded in the selected PDF pages, inserting recognized text at each image's position in page reading order while preserving the PDF text layer. This is separate from automatic scanned-PDF OCR fallback.\n - `shouldParse?: boolean`\n When true, PDF pages are fetched and parsed. When false, PDF pages are skipped entirely (not included in results and not counted as failures).\n - `start?: number`\n First 1-based PDF page to parse. When omitted, parsing starts at the first page.\n\n- `settleAnimations?: boolean`\n When true, waits briefly for CSS and transition animations to settle before extracting each crawled page. Defaults to false. This adds a bit of latency in exchange for more stable output on animated pages.\n\n- `shortenBase64Images?: boolean`\n Truncate base64-encoded image data in the Markdown output\n\n- `stopAfterMs?: number`\n Soft time budget for the crawl in milliseconds. After each scrape, the crawler checks the elapsed time and, if exceeded, returns the pages collected so far instead of continuing. Min: 10000 (10s). Max: 110000 (110s). Default: 80000 (80s).\n\n- `tags?: string[]`\n Optional tags for tracking usage. Up to 20 tags, each 1 to 50 characters.\n\n- `timeoutMS?: number`\n Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes).\n\n- `urlRegex?: string`\n Regex pattern. Only URLs matching this pattern will be followed and scraped.\n\n- `useMainContentOnly?: boolean`\n Extract only the main content, stripping headers, footers, sidebars, and navigation\n\n- `waitForMs?: number`\n Optional browser wait time in milliseconds after initial page load for each crawled page. Min: 0. Max: 30000 (30 seconds). \n\n- `zdr?: 'enabled' | 'disabled'`\n Set to enabled to bypass shared caches and omit request and response content from retained usage logs. Requires zero data retention to be enabled for your organization (contact support@context.dev), otherwise the request fails with ZDR_NOT_ENABLED. Successful ZDR responses include X-Context-ZDR: true.\n\n### Returns\n\n- `{ metadata: { maxCrawlDepth: number; numFailed: number; numSkipped: number; numSucceeded: number; numUrls: number; }; results: { markdown: string; metadata: { crawlDepth: number; finalUrl: string; sourceUrl: string; statusCode: number; success: boolean; title: string; url: string; additionalMeta?: object; alternates?: object[]; author?: string; canonicalUrl?: string; description?: string; favicon?: string; image?: string; jsonLd?: object[]; keywords?: string[]; language?: string; modifiedTime?: string; openGraph?: object; publishedTime?: string; robots?: string; siteName?: string; twitter?: object; }; }[]; key_metadata?: { credits_consumed: number; credits_remaining: number; }; }`\n\n - `metadata: { maxCrawlDepth: number; numFailed: number; numSkipped: number; numSucceeded: number; numUrls: number; }`\n - `results: { markdown: string; metadata: { crawlDepth: number; finalUrl: string; sourceUrl: string; statusCode: number; success: boolean; title: string; url: string; additionalMeta?: object; alternates?: { href: string; hreflang?: string; title?: string; type?: string; }[]; author?: string; canonicalUrl?: string; description?: string; favicon?: string; image?: string; jsonLd?: object[]; keywords?: string[]; language?: string; modifiedTime?: string; openGraph?: object; publishedTime?: string; robots?: string; siteName?: string; twitter?: object; }; }[]`\n - `key_metadata?: { credits_consumed: number; credits_remaining: number; }`\n\n### Example\n\n```typescript\nimport ContextDev from 'context.dev';\n\nconst client = new ContextDev();\n\nconst response = await client.web.webCrawlMd({ url: 'https://example.com' });\n\nconsole.log(response);\n```", + '## web_crawl_md\n\n`client.web.webCrawlMd(url: string, country?: string, excludeSelectors?: string[], followSubdomains?: boolean, includeFrames?: boolean, includeImages?: boolean, includeLinks?: boolean, includeSelectors?: string[], maxAgeMs?: number, maxDepth?: number, maxPages?: number, pdf?: { end?: number; ocr?: boolean; shouldParse?: boolean; start?: number; }, settleAnimations?: boolean, shortenBase64Images?: boolean, stopAfterMs?: number, tags?: string[], timeoutMS?: number, urlRegex?: string, useMainContentOnly?: boolean, waitForMs?: number, zdr?: \'enabled\' | \'disabled\'): { metadata: object; results: object[]; key_metadata?: object; }`\n\n**post** `/web/crawl`\n\nPerforms a crawl starting from a given URL, extracts page content as Markdown, and returns results for all crawled pages.\n\n### Parameters\n\n- `url: string`\n The starting URL for the crawl (must include http:// or https:// protocol)\n\n- `country?: string`\n Fetch the target page through a residential proxy in this country (ISO 3166-1 alpha-2).\n\n- `excludeSelectors?: string[]`\n CSS selectors to remove before each crawled page is converted to Markdown. Applied after includeSelectors. Exclusion takes precedence: an element matching both is removed. Examples: "nav", "footer", ".ad-banner", "[aria-hidden=true]".\n\n- `followSubdomains?: boolean`\n When true, follow links on subdomains of the starting URL\'s domain (e.g. docs.example.com when starting from example.com). www and apex are always treated as equivalent.\n\n- `includeFrames?: boolean`\n When true, the contents of iframes are rendered to Markdown for each crawled page.\n\n- `includeImages?: boolean`\n Include image references in the Markdown output\n\n- `includeLinks?: boolean`\n Preserve hyperlinks in the Markdown output\n\n- `includeSelectors?: string[]`\n CSS selectors. When provided, only matching HTML subtrees (and their descendants) are kept before each crawled page is converted to Markdown. When omitted, the entire document is kept. Examples: "article.main", "#content", "[role=main]".\n\n- `maxAgeMs?: number`\n Return a cached result if a prior scrape for the same parameters exists and is younger than this many milliseconds. Defaults to 1 day (86400000 ms) when omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh.\n\n- `maxDepth?: number`\n Maximum link depth from the starting URL (0 = only the starting page)\n\n- `maxPages?: number`\n Maximum number of pages to crawl. Hard cap: 500.\n\n- `pdf?: { end?: number; ocr?: boolean; shouldParse?: boolean; start?: number; }`\n PDF parsing controls. Use start/end to limit text extraction and embedded-image detection/OCR to an inclusive 1-based page range.\n - `end?: number`\n Last 1-based PDF page to parse. When omitted, parsing ends at the last page. Must be greater than or equal to start when both are provided.\n - `ocr?: boolean`\n When true, detect and OCR images embedded in the selected PDF pages, inserting recognized text at each image\'s position in page reading order while preserving the PDF text layer. This is separate from automatic scanned-PDF OCR fallback.\n - `shouldParse?: boolean`\n When true, PDF pages are fetched and parsed. When false, PDF pages are skipped entirely (not included in results and not counted as failures).\n - `start?: number`\n First 1-based PDF page to parse. When omitted, parsing starts at the first page.\n\n- `settleAnimations?: boolean`\n When true, waits briefly for CSS and transition animations to settle before extracting each crawled page. Defaults to false. This adds a bit of latency in exchange for more stable output on animated pages.\n\n- `shortenBase64Images?: boolean`\n Truncate base64-encoded image data in the Markdown output\n\n- `stopAfterMs?: number`\n Soft time budget for the crawl in milliseconds. After each scrape, the crawler checks the elapsed time and, if exceeded, returns the pages collected so far instead of continuing. Min: 10000 (10s). Max: 110000 (110s). Default: 80000 (80s).\n\n- `tags?: string[]`\n Optional tags for tracking usage. Up to 20 tags, each 1 to 50 characters.\n\n- `timeoutMS?: number`\n Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes).\n\n- `urlRegex?: string`\n Regex pattern. Only URLs matching this pattern will be followed and scraped.\n\n- `useMainContentOnly?: boolean`\n Extract only the main content, stripping headers, footers, sidebars, and navigation\n\n- `waitForMs?: number`\n Optional browser wait time in milliseconds after initial page load for each crawled page. Min: 0. Max: 30000 (30 seconds). \n\n- `zdr?: \'enabled\' | \'disabled\'`\n Set to enabled to bypass shared caches and omit request and response content from retained usage logs. Requires zero data retention to be enabled for your organization (contact support@context.dev), otherwise the request fails with ZDR_NOT_ENABLED. Successful ZDR responses include X-Context-ZDR: true.\n\n### Returns\n\n- `{ metadata: { maxCrawlDepth: number; numFailed: number; numSkipped: number; numSucceeded: number; numUrls: number; }; results: { markdown: string; metadata: { crawlDepth: number; finalUrl: string; sourceUrl: string; statusCode: number; success: boolean; title: string; url: string; additionalMeta?: object; alternates?: object[]; author?: string; canonicalUrl?: string; description?: string; favicon?: string; image?: string; jsonLd?: object[]; keywords?: string[]; language?: string; modifiedTime?: string; openGraph?: object; publishedTime?: string; robots?: string; siteName?: string; twitter?: object; }; }[]; key_metadata?: { credits_consumed: number; credits_remaining: number; }; }`\n\n - `metadata: { maxCrawlDepth: number; numFailed: number; numSkipped: number; numSucceeded: number; numUrls: number; }`\n - `results: { markdown: string; metadata: { crawlDepth: number; finalUrl: string; sourceUrl: string; statusCode: number; success: boolean; title: string; url: string; additionalMeta?: object; alternates?: { href: string; hreflang?: string; title?: string; type?: string; }[]; author?: string; canonicalUrl?: string; description?: string; favicon?: string; image?: string; jsonLd?: object[]; keywords?: string[]; language?: string; modifiedTime?: string; openGraph?: object; publishedTime?: string; robots?: string; siteName?: string; twitter?: object; }; }[]`\n - `key_metadata?: { credits_consumed: number; credits_remaining: number; }`\n\n### Example\n\n```typescript\nimport ContextDev from \'context.dev\';\n\nconst client = new ContextDev();\n\nconst response = await client.web.webCrawlMd({ url: \'https://example.com\' });\n\nconsole.log(response);\n```', perLanguage: { typescript: { method: 'client.web.webCrawlMd', @@ -1930,9 +1930,9 @@ const EMBEDDED_METHODS: MethodEntry[] = [ 'tags?: string;', ], response: - "{ data?: { id: string; credits: { charged: number; estimated: number; }; error: object; errors: object[]; input: { accepted: number; duplicates: number; invalid: number; submitted: number; }; mode: 'scrape' | 'crawl'; progress: { failed: number; pending: number; succeeded: number; }; results: { expires_at: string; files: object[]; }; status: 'queued' | 'running' | 'cancelling' | 'completed' | 'cancelled' | 'failed'; tags: string[]; timing: { completed_at: string; created_at: string; started_at: string; }; type: 'markdown' | 'html'; }[]; has_more?: boolean; key_metadata?: { credits_consumed: number; credits_remaining: number; }; next_cursor?: string; }", + "{ data?: { id: string; crawl: object; credits: { net: number; refunded: number; reserved: number; }; failure: object; format: 'markdown' | 'html'; input: object; mode: 'scrape' | 'crawl'; page_errors: object[]; progress: { failed: number; pending: number; succeeded: number; }; results: { expires_at: string; files: object[]; }; status: 'queued' | 'running' | 'cancelling' | 'completed' | 'cancelled' | 'failed'; tags: string[]; timing: { completed_at: string; created_at: string; started_at: string; }; }[]; has_more?: boolean; key_metadata?: { credits_consumed: number; credits_remaining: number; }; next_cursor?: string; }", markdown: - "## list\n\n`client.batch.list(cursor?: string, limit?: number, q?: string, search_type?: 'exact' | 'prefix', status?: 'queued' | 'running' | 'cancelling' | 'completed' | 'cancelled' | 'failed', tags?: string): { data?: object[]; has_more?: boolean; key_metadata?: object; next_cursor?: string; }`\n\n**get** `/batch/list`\n\nList your batches from newest to oldest. Filter by status or continue with a cursor.\n\n### Parameters\n\n- `cursor?: string`\n Cursor from the previous page.\n\n- `limit?: number`\n Batches per page. Defaults to 25.\n\n- `q?: string`\n Free-text search term, matched against the batch id, crawl source (start URL or sitemap domain), and tags.\n\n- `search_type?: 'exact' | 'prefix'`\n `prefix` for as-you-type prefix matching (default), `exact` for full-token matching.\n\n- `status?: 'queued' | 'running' | 'cancelling' | 'completed' | 'cancelled' | 'failed'`\n Filter by status.\n\n- `tags?: string`\n Comma-separated list of tags to filter by (matches batches having any of them).\n\n### Returns\n\n- `{ data?: { id: string; credits: { charged: number; estimated: number; }; error: object; errors: object[]; input: { accepted: number; duplicates: number; invalid: number; submitted: number; }; mode: 'scrape' | 'crawl'; progress: { failed: number; pending: number; succeeded: number; }; results: { expires_at: string; files: object[]; }; status: 'queued' | 'running' | 'cancelling' | 'completed' | 'cancelled' | 'failed'; tags: string[]; timing: { completed_at: string; created_at: string; started_at: string; }; type: 'markdown' | 'html'; }[]; has_more?: boolean; key_metadata?: { credits_consumed: number; credits_remaining: number; }; next_cursor?: string; }`\n\n - `data?: { id: string; credits: { charged: number; estimated: number; }; error: { code: string; message: string; }; errors: { code: string; count: number; }[]; input: { accepted: number; duplicates: number; invalid: number; submitted: number; }; mode: 'scrape' | 'crawl'; progress: { failed: number; pending: number; succeeded: number; }; results: { expires_at: string; files: { bytes: number; items: number; url: string; }[]; }; status: 'queued' | 'running' | 'cancelling' | 'completed' | 'cancelled' | 'failed'; tags: string[]; timing: { completed_at: string; created_at: string; started_at: string; }; type: 'markdown' | 'html'; }[]`\n - `has_more?: boolean`\n - `key_metadata?: { credits_consumed: number; credits_remaining: number; }`\n - `next_cursor?: string`\n\n### Example\n\n```typescript\nimport ContextDev from 'context.dev';\n\nconst client = new ContextDev();\n\nconst batches = await client.batch.list();\n\nconsole.log(batches);\n```", + "## list\n\n`client.batch.list(cursor?: string, limit?: number, q?: string, search_type?: 'exact' | 'prefix', status?: 'queued' | 'running' | 'cancelling' | 'completed' | 'cancelled' | 'failed', tags?: string): { data?: object[]; has_more?: boolean; key_metadata?: object; next_cursor?: string; }`\n\n**get** `/batch/list`\n\nList your batches from newest to oldest. Filter by status or continue with a cursor.\n\n### Parameters\n\n- `cursor?: string`\n Cursor from the previous page.\n\n- `limit?: number`\n Batches per page. Defaults to 25.\n\n- `q?: string`\n Free-text search term, matched against the batch id, crawl source (start URL or sitemap domain), and tags.\n\n- `search_type?: 'exact' | 'prefix'`\n `prefix` for as-you-type prefix matching (default), `exact` for full-token matching.\n\n- `status?: 'queued' | 'running' | 'cancelling' | 'completed' | 'cancelled' | 'failed'`\n Filter by status.\n\n- `tags?: string`\n Comma-separated list of tags to filter by (matches batches having any of them).\n\n### Returns\n\n- `{ data?: { id: string; crawl: object; credits: { net: number; refunded: number; reserved: number; }; failure: object; format: 'markdown' | 'html'; input: object; mode: 'scrape' | 'crawl'; page_errors: object[]; progress: { failed: number; pending: number; succeeded: number; }; results: { expires_at: string; files: object[]; }; status: 'queued' | 'running' | 'cancelling' | 'completed' | 'cancelled' | 'failed'; tags: string[]; timing: { completed_at: string; created_at: string; started_at: string; }; }[]; has_more?: boolean; key_metadata?: { credits_consumed: number; credits_remaining: number; }; next_cursor?: string; }`\n\n - `data?: { id: string; crawl: { follow_subdomains: boolean; max_depth: number; max_pages: number; source: { type: 'start_url'; url: string; } | { domain: string; type: 'sitemap'; }; url_pattern: string; }; credits: { net: number; refunded: number; reserved: number; }; failure: { code: string; message: string; }; format: 'markdown' | 'html'; input: { duplicates: number; invalid: number; reserved: number; reserved_is_ceiling: boolean; submitted: number; }; mode: 'scrape' | 'crawl'; page_errors: { code: string; count: number; }[]; progress: { failed: number; pending: number; succeeded: number; }; results: { expires_at: string; files: { bytes: number; items: number; url: string; }[]; }; status: 'queued' | 'running' | 'cancelling' | 'completed' | 'cancelled' | 'failed'; tags: string[]; timing: { completed_at: string; created_at: string; started_at: string; }; }[]`\n - `has_more?: boolean`\n - `key_metadata?: { credits_consumed: number; credits_remaining: number; }`\n - `next_cursor?: string`\n\n### Example\n\n```typescript\nimport ContextDev from 'context.dev';\n\nconst client = new ContextDev();\n\nconst batches = await client.batch.list();\n\nconsole.log(batches);\n```", perLanguage: { typescript: { method: 'client.batch.list', @@ -1974,15 +1974,14 @@ const EMBEDDED_METHODS: MethodEntry[] = [ endpoint: '/batch/{batch_id}', httpMethod: 'get', summary: 'Get a batch', - description: - 'Check progress and get download links when the batch finishes. Also returns the rejected-URL list and webhook signing secret from submission, so nothing is lost if the submit response was dropped.', + description: 'Check progress, and get download links once the batch finishes.', stainlessPath: '(resource) batch > (method) retrieve', qualified: 'client.batch.retrieve', params: ['batch_id: string;'], response: - "{ id: string; credits: { charged: number; estimated: number; }; error: { code: string; message: string; }; errors: { code: string; count: number; }[]; input: { accepted: number; duplicates: number; invalid: number; submitted: number; }; invalid_urls: { reason: string; url: string; }[]; mode: 'scrape' | 'crawl'; progress: { failed: number; pending: number; succeeded: number; }; results: { expires_at: string; files: { bytes: number; items: number; url: string; }[]; }; status: 'queued' | 'running' | 'cancelling' | 'completed' | 'cancelled' | 'failed'; tags: string[]; timing: { completed_at: string; created_at: string; started_at: string; }; type: 'markdown' | 'html'; key_metadata?: { credits_consumed: number; credits_remaining: number; }; webhook_secret?: string; }", + "{ id: string; crawl: { follow_subdomains: boolean; max_depth: number; max_pages: number; source: object | object; url_pattern: string; }; credits: { net: number; refunded: number; reserved: number; }; failure: { code: string; message: string; }; format: 'markdown' | 'html'; input: { duplicates: number; invalid: number; reserved: number; reserved_is_ceiling: boolean; submitted: number; }; invalid_urls: { reason: string; url: string; }[]; mode: 'scrape' | 'crawl'; page_errors: { code: string; count: number; }[]; progress: { failed: number; pending: number; succeeded: number; }; results: { expires_at: string; files: { bytes: number; items: number; url: string; }[]; }; status: 'queued' | 'running' | 'cancelling' | 'completed' | 'cancelled' | 'failed'; tags: string[]; timing: { completed_at: string; created_at: string; started_at: string; }; key_metadata?: { credits_consumed: number; credits_remaining: number; }; }", markdown: - "## retrieve\n\n`client.batch.retrieve(batch_id: string): { id: string; credits: object; error: error; errors: error_count[]; input: object; invalid_urls: object[]; mode: 'scrape' | 'crawl'; progress: object; results: object; status: 'queued' | 'running' | 'cancelling' | 'completed' | 'cancelled' | 'failed'; tags: string[]; timing: object; type: 'markdown' | 'html'; key_metadata?: object; webhook_secret?: string; }`\n\n**get** `/batch/{batch_id}`\n\nCheck progress and get download links when the batch finishes. Also returns the rejected-URL list and webhook signing secret from submission, so nothing is lost if the submit response was dropped.\n\n### Parameters\n\n- `batch_id: string`\n ID of the batch to retrieve or cancel.\n\n### Returns\n\n- `{ id: string; credits: { charged: number; estimated: number; }; error: { code: string; message: string; }; errors: { code: string; count: number; }[]; input: { accepted: number; duplicates: number; invalid: number; submitted: number; }; invalid_urls: { reason: string; url: string; }[]; mode: 'scrape' | 'crawl'; progress: { failed: number; pending: number; succeeded: number; }; results: { expires_at: string; files: { bytes: number; items: number; url: string; }[]; }; status: 'queued' | 'running' | 'cancelling' | 'completed' | 'cancelled' | 'failed'; tags: string[]; timing: { completed_at: string; created_at: string; started_at: string; }; type: 'markdown' | 'html'; key_metadata?: { credits_consumed: number; credits_remaining: number; }; webhook_secret?: string; }`\n\n - `id: string`\n - `credits: { charged: number; estimated: number; }`\n - `error: { code: string; message: string; }`\n - `errors: { code: string; count: number; }[]`\n - `input: { accepted: number; duplicates: number; invalid: number; submitted: number; }`\n - `invalid_urls: { reason: string; url: string; }[]`\n - `mode: 'scrape' | 'crawl'`\n - `progress: { failed: number; pending: number; succeeded: number; }`\n - `results: { expires_at: string; files: { bytes: number; items: number; url: string; }[]; }`\n - `status: 'queued' | 'running' | 'cancelling' | 'completed' | 'cancelled' | 'failed'`\n - `tags: string[]`\n - `timing: { completed_at: string; created_at: string; started_at: string; }`\n - `type: 'markdown' | 'html'`\n - `key_metadata?: { credits_consumed: number; credits_remaining: number; }`\n - `webhook_secret?: string`\n\n### Example\n\n```typescript\nimport ContextDev from 'context.dev';\n\nconst client = new ContextDev();\n\nconst batch = await client.batch.retrieve('batch_9f2c8a');\n\nconsole.log(batch);\n```", + "## retrieve\n\n`client.batch.retrieve(batch_id: string): { id: string; crawl: crawl_controls; credits: object; failure: failure; format: 'markdown' | 'html'; input: intake; invalid_urls: object[]; mode: 'scrape' | 'crawl'; page_errors: page_error_count[]; progress: object; results: object; status: 'queued' | 'running' | 'cancelling' | 'completed' | 'cancelled' | 'failed'; tags: string[]; timing: object; key_metadata?: object; }`\n\n**get** `/batch/{batch_id}`\n\nCheck progress, and get download links once the batch finishes.\n\n### Parameters\n\n- `batch_id: string`\n ID of the batch to retrieve or cancel.\n\n### Returns\n\n- `{ id: string; crawl: { follow_subdomains: boolean; max_depth: number; max_pages: number; source: object | object; url_pattern: string; }; credits: { net: number; refunded: number; reserved: number; }; failure: { code: string; message: string; }; format: 'markdown' | 'html'; input: { duplicates: number; invalid: number; reserved: number; reserved_is_ceiling: boolean; submitted: number; }; invalid_urls: { reason: string; url: string; }[]; mode: 'scrape' | 'crawl'; page_errors: { code: string; count: number; }[]; progress: { failed: number; pending: number; succeeded: number; }; results: { expires_at: string; files: { bytes: number; items: number; url: string; }[]; }; status: 'queued' | 'running' | 'cancelling' | 'completed' | 'cancelled' | 'failed'; tags: string[]; timing: { completed_at: string; created_at: string; started_at: string; }; key_metadata?: { credits_consumed: number; credits_remaining: number; }; }`\n\n - `id: string`\n - `crawl: { follow_subdomains: boolean; max_depth: number; max_pages: number; source: { type: 'start_url'; url: string; } | { domain: string; type: 'sitemap'; }; url_pattern: string; }`\n - `credits: { net: number; refunded: number; reserved: number; }`\n - `failure: { code: string; message: string; }`\n - `format: 'markdown' | 'html'`\n - `input: { duplicates: number; invalid: number; reserved: number; reserved_is_ceiling: boolean; submitted: number; }`\n - `invalid_urls: { reason: string; url: string; }[]`\n - `mode: 'scrape' | 'crawl'`\n - `page_errors: { code: string; count: number; }[]`\n - `progress: { failed: number; pending: number; succeeded: number; }`\n - `results: { expires_at: string; files: { bytes: number; items: number; url: string; }[]; }`\n - `status: 'queued' | 'running' | 'cancelling' | 'completed' | 'cancelled' | 'failed'`\n - `tags: string[]`\n - `timing: { completed_at: string; created_at: string; started_at: string; }`\n - `key_metadata?: { credits_consumed: number; credits_remaining: number; }`\n\n### Example\n\n```typescript\nimport ContextDev from 'context.dev';\n\nconst client = new ContextDev();\n\nconst batch = await client.batch.retrieve('batch_9f2c8a');\n\nconsole.log(batch);\n```", perLanguage: { typescript: { method: 'client.batch.retrieve', @@ -2024,15 +2023,14 @@ const EMBEDDED_METHODS: MethodEntry[] = [ endpoint: '/batch/{batch_id}/results', httpMethod: 'get', summary: 'Get batch results', - description: - 'Page through the result records of a finished batch as JSON, in the same order as the downloadable result files. Use this instead of downloading and parsing the NDJSON files yourself.', + description: "Page through a finished batch's results as JSON instead of downloading the NDJSON files.", stainlessPath: '(resource) batch > (method) get_results', qualified: 'client.batch.getResults', params: ['batch_id: string;', 'cursor?: string;', 'limit?: number;'], response: "{ data?: { final_url: string; http_status: number; metadata: { finalUrl: string; sourceUrl: string; additionalMeta?: object; alternates?: object[]; author?: string; canonicalUrl?: string; description?: string; favicon?: string; image?: string; jsonLd?: object[]; keywords?: string[]; language?: string; modifiedTime?: string; openGraph?: object; publishedTime?: string; robots?: string; siteName?: string; title?: string; twitter?: object; }; status: 'ok'; url: string; html?: string; itemId?: string; markdown?: string; meta?: object; } | { error_code: string; message: string; status: 'error'; url: string; itemId?: string; meta?: object; }[]; has_more?: boolean; key_metadata?: { credits_consumed: number; credits_remaining: number; }; next_cursor?: string; }", markdown: - "## get_results\n\n`client.batch.getResults(batch_id: string, cursor?: string, limit?: number): { data?: object | object[]; has_more?: boolean; key_metadata?: object; next_cursor?: string; }`\n\n**get** `/batch/{batch_id}/results`\n\nPage through the result records of a finished batch as JSON, in the same order as the downloadable result files. Use this instead of downloading and parsing the NDJSON files yourself.\n\n### Parameters\n\n- `batch_id: string`\n ID of the batch to retrieve or cancel.\n\n- `cursor?: string`\n next_cursor from the previous page.\n\n- `limit?: number`\n Records per page. Defaults to 25. A page can close early so its payload stays under ~8 MB; rely on next_cursor rather than counting records.\n\n### Returns\n\n- `{ data?: { final_url: string; http_status: number; metadata: { finalUrl: string; sourceUrl: string; additionalMeta?: object; alternates?: object[]; author?: string; canonicalUrl?: string; description?: string; favicon?: string; image?: string; jsonLd?: object[]; keywords?: string[]; language?: string; modifiedTime?: string; openGraph?: object; publishedTime?: string; robots?: string; siteName?: string; title?: string; twitter?: object; }; status: 'ok'; url: string; html?: string; itemId?: string; markdown?: string; meta?: object; } | { error_code: string; message: string; status: 'error'; url: string; itemId?: string; meta?: object; }[]; has_more?: boolean; key_metadata?: { credits_consumed: number; credits_remaining: number; }; next_cursor?: string; }`\n\n - `data?: { final_url: string; http_status: number; metadata: { finalUrl: string; sourceUrl: string; additionalMeta?: object; alternates?: { href: string; hreflang?: string; title?: string; type?: string; }[]; author?: string; canonicalUrl?: string; description?: string; favicon?: string; image?: string; jsonLd?: object[]; keywords?: string[]; language?: string; modifiedTime?: string; openGraph?: object; publishedTime?: string; robots?: string; siteName?: string; title?: string; twitter?: object; }; status: 'ok'; url: string; html?: string; itemId?: string; markdown?: string; meta?: object; } | { error_code: string; message: string; status: 'error'; url: string; itemId?: string; meta?: object; }[]`\n - `has_more?: boolean`\n - `key_metadata?: { credits_consumed: number; credits_remaining: number; }`\n - `next_cursor?: string`\n\n### Example\n\n```typescript\nimport ContextDev from 'context.dev';\n\nconst client = new ContextDev();\n\nconst response = await client.batch.getResults('batch_9f2c8a');\n\nconsole.log(response);\n```", + "## get_results\n\n`client.batch.getResults(batch_id: string, cursor?: string, limit?: number): { data?: object | object[]; has_more?: boolean; key_metadata?: object; next_cursor?: string; }`\n\n**get** `/batch/{batch_id}/results`\n\nPage through a finished batch's results as JSON instead of downloading the NDJSON files.\n\n### Parameters\n\n- `batch_id: string`\n ID of the batch to retrieve or cancel.\n\n- `cursor?: string`\n next_cursor from the previous page.\n\n- `limit?: number`\n Records per page. Defaults to 25. A page can close early so its payload stays under ~8 MB; rely on next_cursor rather than counting records.\n\n### Returns\n\n- `{ data?: { final_url: string; http_status: number; metadata: { finalUrl: string; sourceUrl: string; additionalMeta?: object; alternates?: object[]; author?: string; canonicalUrl?: string; description?: string; favicon?: string; image?: string; jsonLd?: object[]; keywords?: string[]; language?: string; modifiedTime?: string; openGraph?: object; publishedTime?: string; robots?: string; siteName?: string; title?: string; twitter?: object; }; status: 'ok'; url: string; html?: string; itemId?: string; markdown?: string; meta?: object; } | { error_code: string; message: string; status: 'error'; url: string; itemId?: string; meta?: object; }[]; has_more?: boolean; key_metadata?: { credits_consumed: number; credits_remaining: number; }; next_cursor?: string; }`\n\n - `data?: { final_url: string; http_status: number; metadata: { finalUrl: string; sourceUrl: string; additionalMeta?: object; alternates?: { href: string; hreflang?: string; title?: string; type?: string; }[]; author?: string; canonicalUrl?: string; description?: string; favicon?: string; image?: string; jsonLd?: object[]; keywords?: string[]; language?: string; modifiedTime?: string; openGraph?: object; publishedTime?: string; robots?: string; siteName?: string; title?: string; twitter?: object; }; status: 'ok'; url: string; html?: string; itemId?: string; markdown?: string; meta?: object; } | { error_code: string; message: string; status: 'error'; url: string; itemId?: string; meta?: object; }[]`\n - `has_more?: boolean`\n - `key_metadata?: { credits_consumed: number; credits_remaining: number; }`\n - `next_cursor?: string`\n\n### Example\n\n```typescript\nimport ContextDev from 'context.dev';\n\nconst client = new ContextDev();\n\nconst response = await client.batch.getResults('batch_9f2c8a');\n\nconsole.log(response);\n```", perLanguage: { typescript: { method: 'client.batch.getResults', @@ -2080,9 +2078,9 @@ const EMBEDDED_METHODS: MethodEntry[] = [ qualified: 'client.batch.cancel', params: ['batch_id: string;'], response: - "{ id: string; credits: { charged: number; estimated: number; }; error: { code: string; message: string; }; errors: { code: string; count: number; }[]; input: { accepted: number; duplicates: number; invalid: number; submitted: number; }; mode: 'scrape' | 'crawl'; progress: { failed: number; pending: number; succeeded: number; }; results: { expires_at: string; files: { bytes: number; items: number; url: string; }[]; }; status: 'queued' | 'running' | 'cancelling' | 'completed' | 'cancelled' | 'failed'; tags: string[]; timing: { completed_at: string; created_at: string; started_at: string; }; type: 'markdown' | 'html'; key_metadata?: { credits_consumed: number; credits_remaining: number; }; }", + "{ id: string; crawl: { follow_subdomains: boolean; max_depth: number; max_pages: number; source: object | object; url_pattern: string; }; credits: { reserved: number; }; format: 'markdown' | 'html'; input: { duplicates: number; invalid: number; reserved: number; reserved_is_ceiling: boolean; submitted: number; }; mode: 'scrape' | 'crawl'; page_errors: { code: string; count: number; }[]; progress: { failed: number; pending: number; succeeded: number; }; status: 'cancelling'; tags: string[]; timing: { created_at: string; started_at: string; }; key_metadata?: { credits_consumed: number; credits_remaining: number; }; }", markdown: - "## cancel\n\n`client.batch.cancel(batch_id: string): { id: string; credits: object; error: error; errors: error_count[]; input: object; mode: 'scrape' | 'crawl'; progress: object; results: object; status: 'queued' | 'running' | 'cancelling' | 'completed' | 'cancelled' | 'failed'; tags: string[]; timing: object; type: 'markdown' | 'html'; key_metadata?: object; }`\n\n**post** `/batch/{batch_id}/cancel`\n\nStop a batch from starting new pages. In-progress pages finish, and unused credits are refunded.\n\n### Parameters\n\n- `batch_id: string`\n ID of the batch to retrieve or cancel.\n\n### Returns\n\n- `{ id: string; credits: { charged: number; estimated: number; }; error: { code: string; message: string; }; errors: { code: string; count: number; }[]; input: { accepted: number; duplicates: number; invalid: number; submitted: number; }; mode: 'scrape' | 'crawl'; progress: { failed: number; pending: number; succeeded: number; }; results: { expires_at: string; files: { bytes: number; items: number; url: string; }[]; }; status: 'queued' | 'running' | 'cancelling' | 'completed' | 'cancelled' | 'failed'; tags: string[]; timing: { completed_at: string; created_at: string; started_at: string; }; type: 'markdown' | 'html'; key_metadata?: { credits_consumed: number; credits_remaining: number; }; }`\n\n - `id: string`\n - `credits: { charged: number; estimated: number; }`\n - `error: { code: string; message: string; }`\n - `errors: { code: string; count: number; }[]`\n - `input: { accepted: number; duplicates: number; invalid: number; submitted: number; }`\n - `mode: 'scrape' | 'crawl'`\n - `progress: { failed: number; pending: number; succeeded: number; }`\n - `results: { expires_at: string; files: { bytes: number; items: number; url: string; }[]; }`\n - `status: 'queued' | 'running' | 'cancelling' | 'completed' | 'cancelled' | 'failed'`\n - `tags: string[]`\n - `timing: { completed_at: string; created_at: string; started_at: string; }`\n - `type: 'markdown' | 'html'`\n - `key_metadata?: { credits_consumed: number; credits_remaining: number; }`\n\n### Example\n\n```typescript\nimport ContextDev from 'context.dev';\n\nconst client = new ContextDev();\n\nconst response = await client.batch.cancel('batch_9f2c8a');\n\nconsole.log(response);\n```", + "## cancel\n\n`client.batch.cancel(batch_id: string): { id: string; crawl: crawl_controls; credits: object; format: 'markdown' | 'html'; input: intake; mode: 'scrape' | 'crawl'; page_errors: page_error_count[]; progress: object; status: 'cancelling'; tags: string[]; timing: object; key_metadata?: object; }`\n\n**post** `/batch/{batch_id}/cancel`\n\nStop a batch from starting new pages. In-progress pages finish, and unused credits are refunded.\n\n### Parameters\n\n- `batch_id: string`\n ID of the batch to retrieve or cancel.\n\n### Returns\n\n- `{ id: string; crawl: { follow_subdomains: boolean; max_depth: number; max_pages: number; source: object | object; url_pattern: string; }; credits: { reserved: number; }; format: 'markdown' | 'html'; input: { duplicates: number; invalid: number; reserved: number; reserved_is_ceiling: boolean; submitted: number; }; mode: 'scrape' | 'crawl'; page_errors: { code: string; count: number; }[]; progress: { failed: number; pending: number; succeeded: number; }; status: 'cancelling'; tags: string[]; timing: { created_at: string; started_at: string; }; key_metadata?: { credits_consumed: number; credits_remaining: number; }; }`\n\n - `id: string`\n - `crawl: { follow_subdomains: boolean; max_depth: number; max_pages: number; source: { type: 'start_url'; url: string; } | { domain: string; type: 'sitemap'; }; url_pattern: string; }`\n - `credits: { reserved: number; }`\n - `format: 'markdown' | 'html'`\n - `input: { duplicates: number; invalid: number; reserved: number; reserved_is_ceiling: boolean; submitted: number; }`\n - `mode: 'scrape' | 'crawl'`\n - `page_errors: { code: string; count: number; }[]`\n - `progress: { failed: number; pending: number; succeeded: number; }`\n - `status: 'cancelling'`\n - `tags: string[]`\n - `timing: { created_at: string; started_at: string; }`\n - `key_metadata?: { credits_consumed: number; credits_remaining: number; }`\n\n### Example\n\n```typescript\nimport ContextDev from 'context.dev';\n\nconst client = new ContextDev();\n\nconst response = await client.batch.cancel('batch_9f2c8a');\n\nconsole.log(response);\n```", perLanguage: { typescript: { method: 'client.batch.cancel', diff --git a/packages/mcp-server/src/server.ts b/packages/mcp-server/src/server.ts index b8ff8c7..659f676 100644 --- a/packages/mcp-server/src/server.ts +++ b/packages/mcp-server/src/server.ts @@ -28,7 +28,7 @@ export const newMcpServer = async ({ new McpServer( { name: 'context_dev_api', - version: '2.7.0', + version: '2.8.0', }, { instructions: await getInstructions({ stainlessApiKey, customInstructionsPath }), diff --git a/src/client.ts b/src/client.ts index 8da0bbb..740ec43 100644 --- a/src/client.ts +++ b/src/client.ts @@ -34,8 +34,10 @@ import { BatchRetrieveResponse, BatchSubmitParams, BatchSubmitResponse, - Error, - ErrorCount, + CrawlControls, + Failure, + Intake, + PageErrorCount, } from './resources/batch'; import { Brand, @@ -935,8 +937,10 @@ export declare namespace ContextDev { export { Batch as Batch, - type ErrorCount as ErrorCount, - type Error as Error, + type PageErrorCount as PageErrorCount, + type Failure as Failure, + type CrawlControls as CrawlControls, + type Intake as Intake, type BatchRetrieveResponse as BatchRetrieveResponse, type BatchListResponse as BatchListResponse, type BatchCancelResponse as BatchCancelResponse, diff --git a/src/resources/batch.ts b/src/resources/batch.ts index 65e610c..5e3c915 100644 --- a/src/resources/batch.ts +++ b/src/resources/batch.ts @@ -8,9 +8,7 @@ import { path } from '../internal/utils/path'; export class Batch extends APIResource { /** - * Check progress and get download links when the batch finishes. Also returns the - * rejected-URL list and webhook signing secret from submission, so nothing is lost - * if the submit response was dropped. + * Check progress, and get download links once the batch finishes. * * @example * ```ts @@ -51,9 +49,8 @@ export class Batch extends APIResource { } /** - * Page through the result records of a finished batch as JSON, in the same order - * as the downloadable result files. Use this instead of downloading and parsing - * the NDJSON files yourself. + * Page through a finished batch's results as JSON instead of downloading the + * NDJSON files. * * @example * ```ts @@ -91,7 +88,7 @@ export class Batch extends APIResource { /** * Page failures sharing one error code. */ -export interface ErrorCount { +export interface PageErrorCount { /** * Error code for these failures. */ @@ -104,20 +101,113 @@ export interface ErrorCount { } /** - * Why the batch failed. + * A failure of the batch as a whole, distinct from the per-page failures in + * `page_errors`. */ -export interface Error { +export interface Failure { /** - * Batch error code. + * Why the batch itself stopped. */ code: string; /** - * Batch error message. + * Human-readable explanation. */ message: string; } +/** + * The crawl controls as submitted, so the limits requested can be compared against + * what the crawl reached. + */ +export interface CrawlControls { + /** + * Whether links to subdomains were followed. Always false for a sitemap crawl. + */ + follow_subdomains: boolean; + + /** + * Link depth limit. Always 0 for a sitemap crawl, which never follows links off + * its URLs; null when a `start_url` crawl set no limit. + */ + max_depth: number | null; + + /** + * The `maxUrls` submitted with the crawl. A sitemap crawl scrapes only the URLs + * its sitemap actually lists, up to this many, so `input.reserved` is often lower. + */ + max_pages: number; + + /** + * Where the crawl started. + */ + source: CrawlControls.UnionMember0 | CrawlControls.UnionMember1; + + /** + * RE2 pattern URLs had to match to be crawled. Null when the crawl set none. + */ + url_pattern: string | null; +} + +export namespace CrawlControls { + export interface UnionMember0 { + type: 'start_url'; + + /** + * Page the crawl started from. + */ + url: string; + } + + export interface UnionMember1 { + /** + * Domain whose sitemap supplied the pages. + */ + domain: string; + + type: 'sitemap'; + } +} + +/** + * What submission took in, and what it charged for. + */ +export interface Intake { + /** + * URLs dropped before reserving because another entry resolved to the same page. + * Non-zero for sitemap crawls too, whose sitemaps routinely list a page more than + * once. + */ + duplicates: number; + + /** + * URLs from your list rejected as unusable; the same ones are itemised in + * `invalid_urls` at submission. Null for a crawl — a crawl that resolves no usable + * page is rejected outright with a 400 rather than accepted with an empty list. + */ + invalid: number | null; + + /** + * Pages credits were reserved for. Everything else — progress, the refund, the + * completion percentage — is measured against this. + */ + reserved: number; + + /** + * Whether `reserved` is an upper bound the batch may finish under. True only for a + * crawl that follows links, whose reachable page count is unknowable until it + * runs. False for a scrape and for a sitemap crawl, where `reserved` is an exact + * page count. + */ + reserved_is_ceiling: boolean; + + /** + * URLs in the list you sent, before validation and de-duplication. Null for a + * crawl, which is given a source rather than a list. + */ + submitted: number | null; +} + export interface BatchRetrieveResponse { /** * Batch ID used to retrieve or cancel the job. @@ -125,24 +215,32 @@ export interface BatchRetrieveResponse { id: string; /** - * Reserved and used credits. + * The crawl controls as submitted, so the limits requested can be compared against + * what the crawl reached. + */ + crawl: CrawlControls | null; + + /** + * What this batch has done to your credit balance. */ credits: BatchRetrieveResponse.Credits; /** - * Why the batch failed. + * A failure of the batch as a whole, distinct from the per-page failures in + * `page_errors`. */ - error: Error | null; + failure: Failure | null; /** - * Page failures grouped by error code. + * What each page is returned as. Matches `input.data.format` on the submit + * request. */ - errors: Array; + format: 'markdown' | 'html'; /** - * Submission counts. + * What submission took in, and what it charged for. */ - input: BatchRetrieveResponse.Input; + input: Intake; /** * Rejected URLs, up to 100. These are not charged. @@ -150,18 +248,24 @@ export interface BatchRetrieveResponse { invalid_urls: Array; /** - * How pages are selected. + * How pages were selected. Matches `input.mode` on the submit request. */ mode: 'scrape' | 'crawl'; /** - * Current processing counts. Use `status` to check completion. + * Individual page failures grouped by error code, sorted by count. Unrelated to + * `failure`, which is the batch itself failing. + */ + page_errors: Array; + + /** + * Pages attempted so far. Use `status` to check completion. */ progress: BatchRetrieveResponse.Progress; /** - * Download links available when the batch finishes. GET /batch/{batch_id}/results - * serves the same records as paginated JSON. + * Download links, available once the batch reaches a final status and null before + * then. GET /batch/{batch_id}/results serves the same records as paginated JSON. */ results: BatchRetrieveResponse.Results | null; @@ -177,61 +281,34 @@ export interface BatchRetrieveResponse { timing: BatchRetrieveResponse.Timing; - /** - * Output format. - */ - type: 'markdown' | 'html'; - /** * API key usage for this request. */ key_metadata?: BatchRetrieveResponse.KeyMetadata; - - /** - * Webhook signing secret. Also returned by GET /batch/{batch_id}. - */ - webhook_secret?: string; } export namespace BatchRetrieveResponse { /** - * Reserved and used credits. + * What this batch has done to your credit balance. */ export interface Credits { /** - * Credits used by successful pages. + * `reserved` minus `refunded` — what the batch has cost so far. Equal to + * `reserved` until the batch settles. */ - charged: number; + net: number; /** - * Credits reserved when the batch was accepted. + * Credits returned for pages that did not succeed. Stays 0 until the batch reaches + * a final status, then settles in one movement. */ - estimated: number; - } - - /** - * Submission counts. - */ - export interface Input { - /** - * Pages accepted, or the crawl page limit. Credits are reserved for this count. - */ - accepted: number; - - /** - * Duplicate URL and `itemId` pairs skipped. Always 0 for crawls. - */ - duplicates: number; - - /** - * Pages rejected during validation. - */ - invalid: number; + refunded: number; /** - * Pages submitted before validation. For a crawl, the page limit. + * Credits debited from your balance the moment the batch was accepted. This is a + * charge, not a forecast — the whole amount leaves the balance up front. */ - submitted: number; + reserved: number; } export interface InvalidURL { @@ -247,7 +324,7 @@ export namespace BatchRetrieveResponse { } /** - * Current processing counts. Use `status` to check completion. + * Pages attempted so far. Use `status` to check completion. */ export interface Progress { /** @@ -256,8 +333,9 @@ export namespace BatchRetrieveResponse { failed: number; /** - * Accepted pages not yet attempted. Always 0 once the batch completes; a crawl can - * finish under its page limit when the site has no more reachable pages. + * Reserved pages not yet attempted. A cancelled batch keeps reporting the URLs it + * never reached; a crawl whose `input.reserved_is_ceiling` is true reports 0 once + * final, because its unspent budget was never real pages. */ pending: number; @@ -268,8 +346,8 @@ export namespace BatchRetrieveResponse { } /** - * Download links available when the batch finishes. GET /batch/{batch_id}/results - * serves the same records as paginated JSON. + * Download links, available once the batch reaches a final status and null before + * then. GET /batch/{batch_id}/results serves the same records as paginated JSON. */ export interface Results { /** @@ -369,38 +447,52 @@ export namespace BatchListResponse { id: string; /** - * Reserved and used credits. + * The crawl controls as submitted, so the limits requested can be compared against + * what the crawl reached. + */ + crawl: BatchAPI.CrawlControls | null; + + /** + * What this batch has done to your credit balance. */ credits: Data.Credits; /** - * Why the batch failed. + * A failure of the batch as a whole, distinct from the per-page failures in + * `page_errors`. */ - error: BatchAPI.Error | null; + failure: BatchAPI.Failure | null; /** - * Page failures grouped by error code. + * What each page is returned as. Matches `input.data.format` on the submit + * request. */ - errors: Array; + format: 'markdown' | 'html'; /** - * Submission counts. + * What submission took in, and what it charged for. */ - input: Data.Input; + input: BatchAPI.Intake; /** - * How pages are selected. + * How pages were selected. Matches `input.mode` on the submit request. */ mode: 'scrape' | 'crawl'; /** - * Current processing counts. Use `status` to check completion. + * Individual page failures grouped by error code, sorted by count. Unrelated to + * `failure`, which is the batch itself failing. + */ + page_errors: Array; + + /** + * Pages attempted so far. Use `status` to check completion. */ progress: Data.Progress; /** - * Download links available when the batch finishes. GET /batch/{batch_id}/results - * serves the same records as paginated JSON. + * Download links, available once the batch reaches a final status and null before + * then. GET /batch/{batch_id}/results serves the same records as paginated JSON. */ results: Data.Results | null; @@ -415,56 +507,34 @@ export namespace BatchListResponse { tags: Array; timing: Data.Timing; - - /** - * Output format. - */ - type: 'markdown' | 'html'; } export namespace Data { /** - * Reserved and used credits. + * What this batch has done to your credit balance. */ export interface Credits { /** - * Credits used by successful pages. - */ - charged: number; - - /** - * Credits reserved when the batch was accepted. - */ - estimated: number; - } - - /** - * Submission counts. - */ - export interface Input { - /** - * Pages accepted, or the crawl page limit. Credits are reserved for this count. + * `reserved` minus `refunded` — what the batch has cost so far. Equal to + * `reserved` until the batch settles. */ - accepted: number; + net: number; /** - * Duplicate URL and `itemId` pairs skipped. Always 0 for crawls. + * Credits returned for pages that did not succeed. Stays 0 until the batch reaches + * a final status, then settles in one movement. */ - duplicates: number; + refunded: number; /** - * Pages rejected during validation. + * Credits debited from your balance the moment the batch was accepted. This is a + * charge, not a forecast — the whole amount leaves the balance up front. */ - invalid: number; - - /** - * Pages submitted before validation. For a crawl, the page limit. - */ - submitted: number; + reserved: number; } /** - * Current processing counts. Use `status` to check completion. + * Pages attempted so far. Use `status` to check completion. */ export interface Progress { /** @@ -473,8 +543,9 @@ export namespace BatchListResponse { failed: number; /** - * Accepted pages not yet attempted. Always 0 once the batch completes; a crawl can - * finish under its page limit when the site has no more reachable pages. + * Reserved pages not yet attempted. A cancelled batch keeps reporting the URLs it + * never reached; a crawl whose `input.reserved_is_ceiling` is true reports 0 once + * final, because its unspent budget was never real pages. */ pending: number; @@ -485,8 +556,8 @@ export namespace BatchListResponse { } /** - * Download links available when the batch finishes. GET /batch/{batch_id}/results - * serves the same records as paginated JSON. + * Download links, available once the batch reaches a final status and null before + * then. GET /batch/{batch_id}/results serves the same records as paginated JSON. */ export interface Results { /** @@ -556,62 +627,61 @@ export namespace BatchListResponse { export interface BatchCancelResponse { /** - * Batch ID used to retrieve or cancel the job. + * Batch ID. */ id: string; /** - * Reserved and used credits. + * The crawl controls as submitted, so the limits requested can be compared against + * what the crawl reached. */ - credits: BatchCancelResponse.Credits; + crawl: CrawlControls | null; /** - * Why the batch failed. + * What this batch cost so far. */ - error: Error | null; + credits: BatchCancelResponse.Credits; /** - * Page failures grouped by error code. + * What each page is returned as. */ - errors: Array; + format: 'markdown' | 'html'; /** - * Submission counts. + * What submission took in, and what it charged for. */ - input: BatchCancelResponse.Input; + input: Intake; /** - * How pages are selected. + * How pages were selected. */ mode: 'scrape' | 'crawl'; /** - * Current processing counts. Use `status` to check completion. + * Page failures so far, grouped by error code and sorted by count. */ - progress: BatchCancelResponse.Progress; + page_errors: Array; /** - * Download links available when the batch finishes. GET /batch/{batch_id}/results - * serves the same records as paginated JSON. + * How far the batch got before cancellation. */ - results: BatchCancelResponse.Results | null; + progress: BatchCancelResponse.Progress; /** - * Current state. `completed`, `cancelled`, and `failed` are final. + * Always `cancelling`. Work already in flight finishes; the batch reaches + * `cancelled` shortly after. */ - status: 'queued' | 'running' | 'cancelling' | 'completed' | 'cancelled' | 'failed'; + status: 'cancelling'; /** * Tags stored on the batch at submission. */ tags: Array; - timing: BatchCancelResponse.Timing; - /** - * Output format. + * There is no finish time yet — the batch is still winding down. */ - type: 'markdown' | 'html'; + timing: BatchCancelResponse.Timing; /** * API key usage for this request. @@ -621,114 +691,47 @@ export interface BatchCancelResponse { export namespace BatchCancelResponse { /** - * Reserved and used credits. + * What this batch cost so far. */ export interface Credits { /** - * Credits used by successful pages. - */ - charged: number; - - /** - * Credits reserved when the batch was accepted. - */ - estimated: number; - } - - /** - * Submission counts. - */ - export interface Input { - /** - * Pages accepted, or the crawl page limit. Credits are reserved for this count. - */ - accepted: number; - - /** - * Duplicate URL and `itemId` pairs skipped. Always 0 for crawls. - */ - duplicates: number; - - /** - * Pages rejected during validation. - */ - invalid: number; - - /** - * Pages submitted before validation. For a crawl, the page limit. + * Credits debited at submission. The unspent remainder is refunded once the batch + * settles — read `credits.refunded` from GET /batch/{batch_id} then. */ - submitted: number; + reserved: number; } /** - * Current processing counts. Use `status` to check completion. + * How far the batch got before cancellation. */ export interface Progress { /** - * Pages that could not be scraped. + * Pages that could not be scraped before the request landed. */ failed: number; /** - * Accepted pages not yet attempted. Always 0 once the batch completes; a crawl can - * finish under its page limit when the site has no more reachable pages. + * Reserved pages that will now be skipped, and refunded when the batch settles. */ pending: number; /** - * Pages scraped successfully. + * Pages scraped successfully before the request landed. */ succeeded: number; } /** - * Download links available when the batch finishes. GET /batch/{batch_id}/results - * serves the same records as paginated JSON. + * There is no finish time yet — the batch is still winding down. */ - export interface Results { - /** - * When the download URLs expire. - */ - expires_at: string; - - /** - * Result files. Order is not guaranteed. - */ - files: Array; - } - - export namespace Results { - export interface File { - /** - * Compressed file size in bytes. - */ - bytes: number; - - /** - * Results in this file. - */ - items: number; - - /** - * Temporary URL for a gzipped NDJSON file. - */ - url: string; - } - } - export interface Timing { - /** - * When processing finished. Null while active. - */ - completed_at: string | null; - /** * When the batch was created. */ created_at: string; /** - * When processing started. Null while queued. + * When processing started. Null if it was cancelled while still queued. */ started_at: string | null; } @@ -1451,8 +1454,10 @@ export namespace BatchSubmitParams { export declare namespace Batch { export { - type ErrorCount as ErrorCount, - type Error as Error, + type PageErrorCount as PageErrorCount, + type Failure as Failure, + type CrawlControls as CrawlControls, + type Intake as Intake, type BatchRetrieveResponse as BatchRetrieveResponse, type BatchListResponse as BatchListResponse, type BatchCancelResponse as BatchCancelResponse, diff --git a/src/resources/index.ts b/src/resources/index.ts index 114192f..e28f518 100644 --- a/src/resources/index.ts +++ b/src/resources/index.ts @@ -9,8 +9,10 @@ export { } from './ai'; export { Batch, - type ErrorCount, - type Error, + type PageErrorCount, + type Failure, + type CrawlControls, + type Intake, type BatchRetrieveResponse, type BatchListResponse, type BatchCancelResponse, diff --git a/src/resources/web.ts b/src/resources/web.ts index ca71f14..53cf9ec 100644 --- a/src/resources/web.ts +++ b/src/resources/web.ts @@ -2213,9 +2213,8 @@ export interface WebScreenshotParams { colorScheme?: 'light' | 'dark'; /** - * Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - * residential proxy exit location. Must be one of Context.dev's supported - * countries. When provided, Context.dev fetches the target page from that country. + * Fetch the target page through a residential proxy in this country (ISO 3166-1 + * alpha-2). */ country?: | 'ad' @@ -2917,9 +2916,8 @@ export interface WebWebCrawlMdParams { url: string; /** - * Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - * residential proxy exit location. Must be one of Context.dev's supported - * countries. When provided, Context.dev fetches the target page from that country. + * Fetch the target page through a residential proxy in this country (ISO 3166-1 + * alpha-2). */ country?: | 'ad' @@ -3294,9 +3292,8 @@ export interface WebWebScrapeHTMLParams { > | null; /** - * Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - * residential proxy exit location. Must be one of Context.dev's supported - * countries. When provided, Context.dev fetches the target page from that country. + * Fetch the target page through a residential proxy in this country (ISO 3166-1 + * alpha-2). */ country?: | 'ad' @@ -3762,9 +3759,8 @@ export interface WebWebScrapeMdParams { > | null; /** - * Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - * residential proxy exit location. Must be one of Context.dev's supported - * countries. When provided, Context.dev fetches the target page from that country. + * Fetch the target page through a residential proxy in this country (ISO 3166-1 + * alpha-2). */ country?: | 'ad' diff --git a/src/version.ts b/src/version.ts index 9556c21..3981a8d 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1 +1 @@ -export const VERSION = '2.7.0'; // x-release-please-version +export const VERSION = '2.8.0'; // x-release-please-version