From 6d4279b41995d33edf403be39ff71b5d2dd5e43f Mon Sep 17 00:00:00 2001 From: Josef Strzibny Date: Thu, 22 Oct 2026 15:04:08 +0200 Subject: [PATCH 1/2] [Extract] Add extract, map and crawl commands for the Extract API `serpapi extract URL [key=value...]` sends an engine=extract search, so --fields, --jq, output=md, --debug and --timeout work as in search. `serpapi map URL` and `serpapi crawl URL` are shortcuts for `serpapi extract URL mode=map` and `mode=crawl`: same request, parameters and flags. The URL may be given bare or as url=, and must be one absolute http or https URL; anything else is a usage error before any request, as are engine= other than extract, a mode= that contradicts map or crawl, a word without "=", which search would take as q=, and search's --all-pages and --image. Unlike search, the commands need an API key up front, since the Extract API answers nothing without one. A timed-out extraction no longer blames slow engines such as google_ai_mode: extract and map say only how to retry, and crawl suggests --timeout 300 and that each page counts. The README documents the three commands, says map and crawl are shortcuts for mode=map and mode=crawl, and lists crawl among the requests that can outlast the default --timeout. Co-Authored-By: Claude Opus 5.5 --- README.md | 53 ++++++- pkg/api/client.go | 42 +++--- pkg/api/client_test.go | 42 +++++- pkg/cmd/extract.go | 259 +++++++++++++++++++++++++++++++++++ pkg/cmd/extract_test.go | 297 ++++++++++++++++++++++++++++++++++++++++ pkg/cmd/root.go | 37 +++-- tests/e2e_test.go | 36 +++++ 7 files changed, 729 insertions(+), 37 deletions(-) create mode 100644 pkg/cmd/extract.go create mode 100644 pkg/cmd/extract_test.go diff --git a/README.md b/README.md index 233656b..665e615 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,49 @@ serpapi search engine=google_lens --image ./photo.jpg --jq ".visual_matches[:5]" Supported formats are JPG/JPEG, PNG and WebP up to 500 KB. See the [Google Lens image upload docs](https://serpapi.com/google-lens-upload-an-image). +### extract + +Fetches a public web page with the [Extract API](https://serpapi.com/extract-api) and returns its content as structured JSON, or as Markdown with `output=md`. It is a search with `engine=extract`, so `--fields`, `--jq`, `output=`, `async=true` and `archive` work as they do for `search`. The URL must be absolute `http` or `https`; quote it when it has `?` or `&`. It can also be given as `url=URL`. Other parameters are `key=value` pairs sent as-is: + +| Parameter | Does | +|---|---| +| `mode=scrape` | extracts the page itself (default) | +| `mode=map` | lists the website's URLs; shortcut: [`serpapi map`](#map) | +| `mode=crawl` | extracts every page of the website it visits; shortcut: [`serpapi crawl`](#crawl) | +| `include_tags=`, `exclude_tags=` | CSS selectors to keep or drop before parsing | +| `include_markdown=true` | adds the main content as Markdown in `text_content.markdown` | + +```bash +serpapi extract https://example.com +serpapi extract https://example.com output=md +serpapi extract https://example.com --jq '.inline_links[].link' +``` + +It counts one search. The Extract API runs no JavaScript and follows each site's `robots.txt`. Search pages that have their own engine, such as Google or Bing, are redirected to it and return that engine's JSON. Unlike `search`, `extract`, `map` and `crawl` need an API key. + +### map + +`serpapi map URL` is a shortcut for `serpapi extract URL mode=map`: it sends the same request and takes the same parameters and flags. + +Lists a website's URLs, from the links on the page and the website's sitemaps. `include_sitemap=false` keeps to the page's links; `include_subdomains=true` adds other subdomains, such as `blog.example.com` from `www.example.com`. It counts one search. + +```bash +serpapi map https://books.toscrape.com --jq '.urls[].url' +``` + +### crawl + +`serpapi crawl URL` is a shortcut for `serpapi extract URL mode=crawl`: it sends the same request and takes the same parameters and flags. + +Starts at the URL, follows links on the same website and extracts every page it visits, one entry each in `pages`. `max_pages` caps the pages, the first included (default 10, at most 100), and `max_depth` how many links deep it goes (default 2, at most 3; `0` is only the first page). + +```bash +serpapi crawl https://books.toscrape.com max_pages=5 --jq '.pages[].url' +serpapi crawl https://books.toscrape.com max_pages=20 max_depth=1 --timeout 300 +``` + +It counts one search per page it extracts, so the default `max_pages=10` can cost 10; cached and failed pages are free, and `search_metadata.credits_consumed` says what it cost. A crawl can outlast the default 60s `--timeout`, so raise it; `timeout=` is the API's own per-page fetch limit in milliseconds. + ### archive Fetches an earlier search by its `search_metadata.id`, for free. A search made with `async=true` has `search_metadata.status` `Queued` or `Processing` until it is `Success`, so poll. `output=` takes the same values as in search. @@ -143,9 +186,9 @@ Prints the CLI version, as `--version` does. ## Global flags - `--jq `: [jq](https://jqlang.github.io/jq/manual/) filter applied to the JSON result, like `gh --jq`; strings print unquoted and `null` as an empty line, and results stream as they are produced. A filter that prints only `null` gets a warning listing the response's top-level keys. jq is built in, so nothing else needs installing. In bash and zsh, quote the expression with single quotes. -- `--fields `: search only; see [search](#search). +- `--fields `: search, extract, map and crawl only; see [search](#search). - `--api-key `: API key. Prefer `SERPAPI_KEY`, since flags show up in `ps` and shell history. -- `--timeout `: HTTP request timeout in seconds, such as `120`, or a duration, such as `2m` (default `60`; env `SERPAPI_TIMEOUT`). `0` waits indefinitely. Slow engines such as `google_ai_mode`, and searches with `no_cache=true`, can take longer than the default; raise it when a `network_error` says the request timed out, and add `--debug` to see which step stalled. A timed-out search may still finish on SerpApi's side, count against your searches and be available in your search archive. +- `--timeout `: HTTP request timeout in seconds, such as `120`, or a duration, such as `2m` (default `60`; env `SERPAPI_TIMEOUT`). `0` waits indefinitely. Slow engines such as `google_ai_mode`, and searches with `no_cache=true`, can take longer than the default, as can `serpapi crawl`, which extracts every page it visits; raise it when a `network_error` says the request timed out, and add `--debug` to see which step stalled. A timed-out search may still finish on SerpApi's side, count against your searches and be available in your search archive. ```bash # Give an AI Mode query up to two minutes serpapi search --timeout 120 engine=google_ai_mode q="..." @@ -185,7 +228,7 @@ The API key comes from the first of: 3. `SERPAPI_API_KEY` 4. the key saved by `serpapi login` -Without a key, `search` still runs, but the API answers only a few demo queries such as `q=coffee` and returns 401 for the rest. `account` and `archive` need a key. +Without a key, `search` still runs, but the API answers only a few demo queries such as `q=coffee` and returns 401 for the rest. `account`, `archive`, `extract`, `map` and `crawl` need a key. `serpapi login` saves the key, readable only by you, to `serpapi/config.toml` in the OS config directory: `$XDG_CONFIG_HOME` or `~/.config` on Linux, `~/Library/Application Support` on macOS, `%AppData%` on Windows. Without an absolute `HOME` (`%AppData%` on Windows) there is no config file: `login` exits 2, and other commands need `--api-key` or `SERPAPI_KEY`. @@ -193,7 +236,7 @@ Without a key, `search` still runs, but the API answers only a few demo queries api_key = "your_serpapi_key_here" ``` -A config file that can't be read or isn't valid TOML is reported by path and line, never by content: `search` warns and runs without a key, while `account` and `archive` stop. `serpapi login` rewrites it. +A config file that can't be read or isn't valid TOML is reported by path and line, never by content: `search` warns and runs without a key, while `account`, `archive`, `extract`, `map` and `crawl` stop. `serpapi login` rewrites it. ## For AI agents @@ -217,7 +260,7 @@ A config file that can't be read or isn't valid TOML is reported by path and lin | 1 | `network_error` | No answer, such as a timeout. | | 130 | `interrupted` | Ctrl-C or SIGTERM stopped it before it finished, after printing what had arrived, such as the `--all-pages` pages fetched so far. | -- **Cost**: each successful search counts against your monthly searches, including each `--all-pages` page; cached, errored and failed ones don't, though errored ones count toward the hourly limit ([pricing FAQ](https://serpapi.com/pricing)). An identical search, `--fields` included, repeated within an hour comes from the cache. `no_cache=true` forces a new, counted search; cached results are at most an hour old, so add it only when that is too old. `zero_trace=true` (Enterprise) stores nothing, so it skips the cache too and leaves no search for `archive` to read. `account`, `locations` and `archive` are free, and so is an `image` upload, though it needs searches left this month and this hour, and has its own hourly limit, `account_rate_limit_per_hour` in `serpapi account`. +- **Cost**: each successful search counts against your monthly searches, including each `--all-pages` page; cached, errored and failed ones don't, though errored ones count toward the hourly limit ([pricing FAQ](https://serpapi.com/pricing)). An identical search, `--fields` included, repeated within an hour comes from the cache. `no_cache=true` forces a new, counted search; cached results are at most an hour old, so add it only when that is too old. `zero_trace=true` (Enterprise) stores nothing, so it skips the cache too and leaves no search for `archive` to read. `account`, `locations` and `archive` are free, and so is an `image` upload, though it needs searches left this month and this hour, and has its own hourly limit, `account_rate_limit_per_hour` in `serpapi account`. `crawl` counts one search per page it extracts. - **No prompts without a terminal.** Only `serpapi login` and an image given as `-` (`serpapi image -`, `search --image -`) read stdin. `login` waits for the first line, so pipe the key in (`printf %s "$KEY" | serpapi login`) or, simpler, set `SERPAPI_KEY`; an image is read until stdin closes. ## Links diff --git a/pkg/api/client.go b/pkg/api/client.go index 53129e8..7edbc2a 100644 --- a/pkg/api/client.go +++ b/pkg/api/client.go @@ -172,7 +172,7 @@ func (c *Client) do(req *http.Request, endpoint string, tr *tracer) ([]byte, err resp, err := c.http.Do(req) if err != nil { tr.logf("Request failed: %v", err) - return nil, c.networkError(endpoint, err) + return nil, c.networkError(endpoint, req.URL.Query(), err) } defer resp.Body.Close() tr.logResponse(resp) @@ -180,7 +180,7 @@ func (c *Client) do(req *http.Request, endpoint string, tr *tracer) ([]byte, err body, err := io.ReadAll(io.LimitReader(resp.Body, int64(maxResponseBytes)+1)) if err != nil { tr.logf("Read body failed: %v", err) - return nil, c.networkError(endpoint, err) + return nil, c.networkError(endpoint, req.URL.Query(), err) } tr.logf("Body received: %d bytes", len(body)) @@ -197,8 +197,10 @@ func (c *Client) do(req *http.Request, endpoint string, tr *tracer) ([]byte, err return body, nil } -// networkError describes a transport failure without the request URL, which contains the API key. -func (c *Client) networkError(endpoint string, err error) error { +// networkError describes a transport failure without the request URL, which +// contains the API key. query, the request's query parameters, tailors the +// timeout advice to the engine. +func (c *Client) networkError(endpoint string, query url.Values, err error) error { cause := err var urlErr *url.Error if errors.As(err, &urlErr) { @@ -210,21 +212,27 @@ func (c *Client) networkError(endpoint string, err error) error { if c.debug == nil { debugHint = "; --debug shows which step stalled" } - // An upload has no engine, search or archive entry to advise on. - if endpoint == "/image" { - return &clierrors.NetworkError{ - Message: fmt.Sprintf("Request to %s timed out after %s: retry with a higher --timeout, such as --timeout %d, or 0 to wait indefinitely%s", - endpoint, seconds(c.http.Timeout), suggestedTimeout(c.http.Timeout), debugHint), - Cause: cause, - } + timedOut := fmt.Sprintf("Request to %s timed out after %s", endpoint, seconds(c.http.Timeout)) + retry := func(suggested int64) string { + return fmt.Sprintf("retry with a higher --timeout, such as --timeout %d, or 0 to wait indefinitely%s", suggested, debugHint) } - return &clierrors.NetworkError{ - Message: fmt.Sprintf("Request to %s timed out after %s. Slow engines, such as google_ai_mode, and no_cache=true can take longer: "+ - "retry with a higher --timeout, such as --timeout %d, or 0 to wait indefinitely%s. "+ - "The search may still finish on SerpApi's side, count against your searches and be available in your search archive", - endpoint, seconds(c.http.Timeout), suggestedTimeout(c.http.Timeout), debugHint), - Cause: cause, + const onSerpApi = " may still finish on SerpApi's side, count against your searches and be available in your search archive" + var message string + switch { + // An upload has no engine, search or archive entry to advise on. + case endpoint == "/image": + message = timedOut + ": " + retry(suggestedTimeout(c.http.Timeout)) + case strings.EqualFold(query.Get("engine"), "extract") && strings.EqualFold(query.Get("mode"), "crawl"): + message = timedOut + ". A crawl extracts every page it visits, so it takes longer than one page: " + + retry(max(suggestedTimeout(c.http.Timeout), 300)) + + ". The crawl may still finish on SerpApi's side, count one search per page and be available in your search archive" + case strings.EqualFold(query.Get("engine"), "extract"): + message = timedOut + ": " + retry(suggestedTimeout(c.http.Timeout)) + ". The extraction" + onSerpApi + default: + message = timedOut + ". Slow engines, such as google_ai_mode, and no_cache=true can take longer: " + + retry(suggestedTimeout(c.http.Timeout)) + ". The search" + onSerpApi } + return &clierrors.NetworkError{Message: message, Cause: cause} } return &clierrors.NetworkError{ Message: fmt.Sprintf("Request to %s failed: %v", endpoint, cause), diff --git a/pkg/api/client_test.go b/pkg/api/client_test.go index 918a013..f850b47 100644 --- a/pkg/api/client_test.go +++ b/pkg/api/client_test.go @@ -406,7 +406,7 @@ func (osTimeout) Temporary() bool { return true } // system timeout, such as a TCP connect timeout, is reported as is. func TestOSTimeoutWithoutAClientLimit(t *testing.T) { c := New("secret", WithTimeout(0)) - err := c.networkError("/search.json", &url.Error{Op: "Get", URL: "https://serpapi.com/search.json?api_key=secret", Err: osTimeout{}}) + err := c.networkError("/search.json", nil, &url.Error{Op: "Get", URL: "https://serpapi.com/search.json?api_key=secret", Err: osTimeout{}}) var netErr *clierrors.NetworkError if !errors.As(err, &netErr) || netErr.Message != "Request to /search.json failed: i/o timeout" { t.Errorf("got %v, want the OS timeout without --timeout advice", err) @@ -421,7 +421,7 @@ func TestTimeoutAdviceRaisesTheLimit(t *testing.T) { 120 * time.Second: "--timeout 240,", 301 * time.Second: "--timeout 602,", } { - err := New("secret", WithTimeout(limit)).networkError("/search.json", osTimeout{}) + err := New("secret", WithTimeout(limit)).networkError("/search.json", nil, osTimeout{}) var netErr *clierrors.NetworkError if !errors.As(err, &netErr) || !strings.Contains(netErr.Message, want) { t.Errorf("limit %s: got %v, want it to contain %q", limit, err, want) @@ -437,7 +437,7 @@ func TestTimeoutAdvicePointsToDebugWhenItIsOff(t *testing.T) { WithDebug(io.Discard, false)(c) } var netErr *clierrors.NetworkError - if err := c.networkError("/search.json", osTimeout{}); !errors.As(err, &netErr) { + if err := c.networkError("/search.json", nil, osTimeout{}); !errors.As(err, &netErr) { t.Fatalf("debug %v: got %v, want a NetworkError", debug, err) } if suggests := strings.Contains(netErr.Message, "--debug shows which step stalled"); suggests == debug { @@ -631,8 +631,42 @@ func TestUploadTimeoutAdviceLeavesOutTheSearch(t *testing.T) { archivePath("abc"): search(archivePath("abc")), } { var netErr *clierrors.NetworkError - if err := New("secret", WithTimeout(50*time.Millisecond)).networkError(endpoint, osTimeout{}); !errors.As(err, &netErr) || netErr.Message != want { + if err := New("secret", WithTimeout(50*time.Millisecond)).networkError(endpoint, nil, osTimeout{}); !errors.As(err, &netErr) || netErr.Message != want { t.Errorf("%s: got %v\nwant %s", endpoint, err, want) } } } + +// The Extract API has no slow engines to blame: its advice is about the +// extraction, and a crawl's about the pages it extracts. +func TestExtractTimeoutAdvice(t *testing.T) { + retry := func(suggested string) string { + return "retry with a higher --timeout, such as --timeout " + suggested + ", or 0 to wait indefinitely; --debug shows which step stalled" + } + extraction := "Request to /search.json timed out after 50ms: " + retry("120") + + ". The extraction may still finish on SerpApi's side, count against your searches and be available in your search archive" + crawl := "Request to /search.json timed out after 50ms. A crawl extracts every page it visits, so it takes longer than one page: " + retry("300") + + ". The crawl may still finish on SerpApi's side, count one search per page and be available in your search archive" + for _, tc := range []struct { + query url.Values + want string + }{ + {url.Values{"engine": {"extract"}}, extraction}, + {url.Values{"engine": {"extract"}, "mode": {"map"}}, extraction}, + {url.Values{"engine": {"Extract"}, "mode": {"scrape"}}, extraction}, + {url.Values{"engine": {"extract"}, "mode": {"crawl"}}, crawl}, + {url.Values{"engine": {"EXTRACT"}, "mode": {"Crawl"}}, crawl}, + } { + var netErr *clierrors.NetworkError + if err := New("secret", WithTimeout(50*time.Millisecond)).networkError("/search.json", tc.query, osTimeout{}); !errors.As(err, &netErr) || netErr.Message != tc.want { + t.Errorf("%v: got %v\nwant %s", tc.query, err, tc.want) + } + } + + // A crawl that outlasted a long limit is told to double it. + query := url.Values{"engine": {"extract"}, "mode": {"crawl"}} + var netErr *clierrors.NetworkError + if err := New("secret", WithTimeout(400*time.Second)).networkError("/search.json", query, osTimeout{}); !errors.As(err, &netErr) || !strings.Contains(netErr.Message, "--timeout 800,") { + t.Errorf("got %v, want --timeout 800", err) + } +} diff --git a/pkg/cmd/extract.go b/pkg/cmd/extract.go new file mode 100644 index 0000000..e9e1b48 --- /dev/null +++ b/pkg/cmd/extract.go @@ -0,0 +1,259 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "net/url" + "strings" + + "github.com/spf13/cobra" + + clierrors "github.com/serpapi/serpapi-cli/pkg/errors" + "github.com/serpapi/serpapi-cli/pkg/output" + "github.com/serpapi/serpapi-cli/pkg/params" +) + +var extractCmd = &cobra.Command{ + Use: "extract URL [PARAMS...]", + Short: "Extract a web page as JSON or Markdown with the Extract API", + Long: `Extract a public web page with the SerpApi Extract API (engine=extract): +https://serpapi.com/extract-api + +The URL must be absolute http or https; quote it when it has ? or &. It may +also be given as url=URL. Other parameters are key=value pairs sent as-is: + mode=scrape the page itself (default) + mode=map the website's URLs; same as 'serpapi map' + mode=crawl every page of the website it visits; same as 'serpapi crawl' + include_tags=, exclude_tags= CSS selectors to keep or drop + include_markdown=true add the main content as Markdown to JSON + output=md only the main content, as Markdown +Search pages with their own engine, such as Google or Bing, are redirected to +it and return that engine's JSON. The Extract API runs no JavaScript and obeys +robots.txt, so a page that disallows it returns an error. + +--fields and --jq work as in search, e.g. --jq .text_content.full. Entries +with a URL carry a serpapi_extract_link for extracting that URL next. + +Cost: scrape and map count one search each; crawl counts one per page it +extracts. Cached and failed ones are free, as in search.`, + Example: ` serpapi extract https://example.com + serpapi extract https://example.com output=md + serpapi extract https://example.com --jq ".inline_links[].link" + serpapi extract https://example.com include_tags=main --jq .text_content.full`, + Args: cobra.ArbitraryArgs, + RunE: runExtractMode(""), +} + +var mapCmd = &cobra.Command{ + Use: "map URL [PARAMS...]", + Short: "List a website's URLs with the Extract API (extract mode=map)", + Long: `List a website's URLs, from the links on the page at URL and the website's +sitemaps. Same as 'serpapi extract URL mode=map'; see 'serpapi extract --help'. + +Parameters are key=value pairs sent as-is: + include_sitemap=false only the page's links, not /sitemap.xml and the + sitemaps robots.txt lists + include_subdomains=true also other subdomains, such as blog.example.com + from www.example.com +URLs that robots.txt disallows are left out. Each entry in urls has a +serpapi_extract_link for extracting it next. + +Cost: one search.`, + Example: ` serpapi map https://books.toscrape.com + serpapi map https://books.toscrape.com --jq ".urls[].url" + serpapi map https://example.com include_sitemap=false include_subdomains=true`, + Args: cobra.ArbitraryArgs, + RunE: runExtractMode("map"), +} + +var crawlCmd = &cobra.Command{ + Use: "crawl URL [PARAMS...]", + Short: "Extract a website's pages by following its links (extract mode=crawl)", + Long: `Start at URL, follow links on the same website and extract every page +visited, as 'serpapi extract' does one page. Same as +'serpapi extract URL mode=crawl'; see 'serpapi extract --help'. + +Parameters are key=value pairs sent as-is: + max_pages=N pages to extract, the first included (default 10, + at most 100) + max_depth=N links deep to follow; 0 is only the first page + (default 2, at most 3) + include_subdomains=true also follow links to other subdomains +Links that robots.txt disallows are not followed. The result has one entry +per page in pages. + +Cost: one search per page extracted, the first included, so the default +max_pages=10 can cost 10; cached and failed pages are free. +search_metadata.credits_consumed says what it cost. A crawl can outlast the +60s --timeout: raise it, e.g. --timeout 300. timeout= is the Extract API's own +per-page fetch limit in milliseconds.`, + Example: ` serpapi crawl https://books.toscrape.com max_pages=5 --jq ".pages[].url" + serpapi crawl https://books.toscrape.com max_pages=20 max_depth=1 --timeout 300 + serpapi crawl https://example.com/docs/ --fields "pages[].{url, text_content}"`, + Args: cobra.ArbitraryArgs, + RunE: runExtractMode("crawl"), +} + +func init() { + rootCmd.AddCommand(extractCmd, mapCmd, crawlCmd) +} + +// isExtractCommand reports whether cmd sends an Extract API search. +func isExtractCommand(cmd *cobra.Command) bool { + return cmd == extractCmd || cmd == mapCmd || cmd == crawlCmd +} + +// runExtractMode returns the RunE of an Extract API command; a non-empty mode +// is sent as mode=, as map and crawl do. +func runExtractMode(mode string) func(*cobra.Command, []string) error { + return func(cmd *cobra.Command, args []string) error { return runExtract(cmd, args, mode) } +} + +func runExtract(cmd *cobra.Command, args []string, mode string) error { + p, err := extractParams(cmd, args, mode) + if err != nil { + return err + } + raw := false + if o, ok := p["output"]; ok { + if raw, err = isRawOutput(o); err != nil { + return err + } + } + if raw && jqFlag != "" { + return &clierrors.UsageError{Message: "--jq needs JSON, but output=" + p["output"] + " is not JSON"} + } + restricted := params.ApplyFields(p, fieldsFlag, "error") != "" + + // Unlike search, the Extract API answers nothing without a key. + apiKey, err := resolveAPIKey() + if err != nil { + return err + } + client := newClient(apiKey) + ctx := cmd.Context() + label := extractLabel(p["mode"]) + + if raw { + body, err := withSpinner(label, func() ([]byte, error) { return client.SearchRaw(ctx, p) }) + if err != nil { + return err + } + return outputError(output.PrintRaw(cmd.OutOrStdout(), body)) + } + result, err := withSpinner(label, func() (json.RawMessage, error) { return client.Search(ctx, p) }) + if err != nil { + return err + } + if restricted && isEmptyObject(result) { + warnNothingMatched(cmd) + } + return handleOutput(cmd, result, searchHint(p)) +} + +// extractParams turns the arguments into the parameters of an Extract API +// search: the one URL, given bare or as url=, plus key=value pairs, and mode +// when the command sets one. +func extractParams(cmd *cobra.Command, args []string, mode string) (map[string]string, error) { + var urls []string + var parsed []params.Param + for _, a := range args { + if isURLArg(a) { + urls = append(urls, a) + continue + } + // search takes a bare word as q=; here it is a parameter missing its "=". + if !strings.Contains(a, "=") { + return nil, &clierrors.UsageError{Message: fmt.Sprintf("Unexpected argument %q: API parameters are key=value pairs, e.g. %s", + a, extractParamExample(cmd))} + } + p, err := params.ParseParam(a) + if err != nil { + return nil, err + } + // Two url= are two URLs, refused below, not a repeat to warn about. + if p.Key == "url" { + urls = append(urls, p.Value) + continue + } + parsed = append(parsed, p) + } + warnRepeated(cmd, parsed) + p := params.ParamsToMap(parsed) + + name := cmd.Name() + switch len(urls) { + case 0: + return nil, &clierrors.UsageError{Message: "No URL given, e.g. serpapi " + name + " https://example.com"} + case 1: + default: + msg := fmt.Sprintf("%s takes one URL, got %d: %s", name, len(urls), strings.Join(urls, ", ")) + if mode != "crawl" { + msg += "; to extract several pages of one website, use 'serpapi crawl URL'" + } + return nil, &clierrors.UsageError{Message: msg} + } + if err := checkExtractURL(urls[0]); err != nil { + return nil, err + } + if e, ok := p["engine"]; ok && !strings.EqualFold(e, "extract") { + return nil, &clierrors.UsageError{Message: name + " always uses engine=extract; for engine=" + e + ", use 'serpapi search engine=" + e + "'"} + } + if m, ok := p["mode"]; ok && mode != "" && !strings.EqualFold(m, mode) { + return nil, &clierrors.UsageError{Message: name + " always uses mode=" + mode + "; drop mode=" + m + + ", or use 'serpapi extract URL mode=" + m + "'"} + } + p["engine"] = "extract" + p["url"] = urls[0] + if mode != "" { + p["mode"] = mode + } + return p, nil +} + +// isURLArg reports whether arg is meant as the URL rather than a key=value +// parameter: a URL has ":", "/", "." or "?" before any "=" of its query, and +// no parameter name does. An empty argument, as from an unset "$URL", counts +// as the URL, so it is reported as an invalid one. +func isURLArg(arg string) bool { + key, _, _ := strings.Cut(arg, "=") + return arg == "" || strings.ContainsAny(key, ":/.?") +} + +// extractParamExample is a parameter that cmd takes, for usage hints. It goes +// by name: comparing with mapCmd would make its initialization refer to itself. +func extractParamExample(cmd *cobra.Command) string { + switch cmd.Name() { + case "map": + return "include_sitemap=false" + case "crawl": + return "max_pages=5 max_depth=1" + } + return "include_tags=main" +} + +// checkExtractURL refuses, before any request, a URL the Extract API would: +// it takes only absolute http and https URLs. +func checkExtractURL(raw string) error { + u, err := url.Parse(raw) + if err == nil && (u.Scheme == "http" || u.Scheme == "https") && u.Host != "" { + return nil + } + msg := fmt.Sprintf("Invalid URL %q: the Extract API takes an absolute http or https URL", raw) + if !strings.Contains(raw, "://") { + if u, err := url.Parse("https://" + raw); err == nil && strings.Contains(u.Hostname(), ".") { + msg += ", e.g. https://" + raw + } + } + return &clierrors.UsageError{Message: msg} +} + +func extractLabel(mode string) string { + switch strings.ToLower(mode) { + case "map": + return "Mapping website..." + case "crawl": + return "Crawling website..." + } + return "Extracting page..." +} diff --git a/pkg/cmd/extract_test.go b/pkg/cmd/extract_test.go new file mode 100644 index 0000000..b9a21e9 --- /dev/null +++ b/pkg/cmd/extract_test.go @@ -0,0 +1,297 @@ +package cmd + +import ( + "encoding/json" + "net/http" + "strings" + "testing" + "time" +) + +const extractKey = "extract_test_key" + +func TestExtractSendsTheURLAsAnExtractSearch(t *testing.T) { + const page = "https://example.com/a?x=1&y=2" + for _, tc := range []struct { + name string + args []string + }{ + {"bare URL", []string{page}}, + {"url=", []string{"url=" + page}}, + {"bare URL after params", []string{"mode=map", "include_sitemap=false", page}}, + {"engine=extract given", []string{"engine=extract", page}}, + {"engine in another case", []string{"engine=Extract", page}}, + } { + t.Run(tc.name, func(t *testing.T) { + reqs := fakeAPI(t, respond(`{"search_metadata":{"id":"e1"},"page_information":{"title":"Example Domain"}}`)) + + res := execute(t, "", append([]string{"--api-key", extractKey, "extract"}, tc.args...)...) + + wantWarnings(t, wantOK(t, res)) + if !strings.Contains(res.stdout, `"title": "Example Domain"`) { + t.Errorf("stdout = %q, want the indented response", res.stdout) + } + u := reqs.only(t) + wantQuery(t, u, "/search.json", map[string]string{"engine": "extract", "url": page, "api_key": extractKey}) + if q := u.Query(); q.Has("q") || q.Has("x") || q.Has("y") { + t.Errorf("query = %s, want the URL's own query kept inside url=", u.RawQuery) + } + }) + } +} + +func TestExtractSendsParamsAndFields(t *testing.T) { + reqs := fakeAPI(t, respond(`{"pages":[{"url":"https://example.com/"}]}`)) + + res := execute(t, "", "--api-key", extractKey, "extract", "https://example.com/", "mode=crawl", "max_pages=5", + "--fields", "pages[].url", "--jq", ".pages[0].url") + + wantOK(t, res) + if got := strings.TrimSpace(res.stdout); got != "https://example.com/" { + t.Errorf("stdout = %q, want the --jq result", got) + } + wantQuery(t, reqs.only(t), "/search.json", map[string]string{ + "engine": "extract", "mode": "crawl", "max_pages": "5", "json_restrictor": "pages[].url,error", + }) +} + +func TestExtractPrintsMarkdownVerbatim(t *testing.T) { + reqs := fakeAPI(t, respond("---\nsearch_parameters:\n engine: extract\n---\n\nThis domain is for use in documentation examples.\n")) + + res := execute(t, "", "--api-key", extractKey, "extract", "https://example.com/", "output=md") + + wantOK(t, res) + if res.stdout != "---\nsearch_parameters:\n engine: extract\n---\n\nThis domain is for use in documentation examples.\n" { + t.Errorf("stdout = %q, want the Markdown as sent", res.stdout) + } + wantQuery(t, reqs.only(t), "/search.json", map[string]string{"engine": "extract", "output": "md"}) +} + +func TestExtractReportsAPIErrors(t *testing.T) { + fakeAPI(t, respond(`{"search_metadata":{"status":"Error"},"error":"URL is disallowed by robots.txt."}`)) + + res := execute(t, "", "--api-key", extractKey, "extract", "https://example.com/private") + + if status := wantError(t, res, 1, "api_error", "URL is disallowed by robots.txt.")["status"]; status != float64(http.StatusOK) { + t.Errorf("status = %v, want 200", status) + } +} + +// Each of these is wrong before any request, so none may cost a search. +func TestExtractUsageErrorsAreUnsent(t *testing.T) { + for _, tc := range []struct { + name string + args []string + msg string + }{ + {"no URL", nil, "No URL given, e.g. serpapi extract https://example.com"}, + {"params but no URL", []string{"mode=map"}, "No URL given"}, + {"two URLs", []string{"https://a.example", "https://b.example"}, "extract takes one URL, got 2: https://a.example, https://b.example"}, + {"URL and url=", []string{"https://a.example", "url=https://b.example"}, "extract takes one URL, got 2"}, + {"two url=", []string{"url=https://a.example", "url=https://b.example"}, "extract takes one URL, got 2: https://a.example, https://b.example"}, + {"parameter without =", []string{"https://example.com", "main"}, + `Unexpected argument "main": API parameters are key=value pairs, e.g. include_tags=main`}, + {"no scheme", []string{"example.com/page"}, `Invalid URL "example.com/page": the Extract API takes an absolute http or https URL, e.g. https://example.com/page`}, + {"other scheme", []string{"ftp://example.com/file"}, "takes an absolute http or https URL"}, + {"relative path", []string{"/about"}, `Invalid URL "/about"`}, + {"empty url=", []string{"url="}, `Invalid URL ""`}, + {"other engine", []string{"https://example.com", "engine=google"}, "for engine=google, use 'serpapi search engine=google'"}, + {"key as a parameter", []string{"https://example.com", "api_key=secret"}, "Pass the API key with --api-key"}, + {"jq with Markdown", []string{"https://example.com", "output=md", "--jq", ".x"}, "--jq needs JSON, but output=md is not JSON"}, + {"unknown output", []string{"https://example.com", "output=xml"}, "Unknown output=xml"}, + {"parameter as a flag", []string{"https://example.com", "--mode", "map"}, "unknown flag: --mode; API parameters are key=value pairs, e.g. mode=VALUE"}, + {"one-letter flag", []string{"https://example.com", "-m", "map"}, "API parameters are key=value pairs, e.g. include_tags=main"}, + } { + t.Run(tc.name, func(t *testing.T) { + fakeAPI(t, nil) + res := execute(t, "", append([]string{"--api-key", extractKey, "extract"}, tc.args...)...) + wantError(t, res, 2, "usage_error", tc.msg) + if strings.Contains(res.stderr, "given more than once") { + t.Errorf("stderr = %q, want the error alone, not a warning that the last value is used", res.stderr) + } + }) + } +} + +// Unlike search, the Extract API answers nothing without a key, so the CLI +// stops before sending. +func TestExtractNeedsAKeyBeforeSending(t *testing.T) { + fakeAPI(t, nil) + + res := execute(t, "", "extract", "https://example.com") + + wantError(t, res, 2, "usage_error", "No API key") +} + +// map and crawl are extract with mode= preset. +func TestMapAndCrawlSendTheirMode(t *testing.T) { + for _, tc := range []struct { + name string + args []string + want map[string]string + }{ + {"map", []string{"map", "https://example.com", "include_sitemap=false"}, + map[string]string{"mode": "map", "include_sitemap": "false"}}, + {"crawl", []string{"crawl", "url=https://example.com", "max_pages=5", "max_depth=1"}, + map[string]string{"mode": "crawl", "max_pages": "5", "max_depth": "1"}}, + {"same mode given", []string{"crawl", "https://example.com", "mode=crawl"}, map[string]string{"mode": "crawl"}}, + {"same mode in another case", []string{"map", "https://example.com", "mode=MAP"}, map[string]string{"mode": "map"}}, + {"crawl with fields", []string{"crawl", "https://example.com", "--fields", "pages[].url"}, + map[string]string{"mode": "crawl", "json_restrictor": "pages[].url,error"}}, + } { + t.Run(tc.name, func(t *testing.T) { + reqs := fakeAPI(t, respond(`{"search_metadata":{"id":"e1"}}`)) + + res := execute(t, "", append([]string{"--api-key", extractKey}, tc.args...)...) + + wantWarnings(t, wantOK(t, res)) + u := reqs.only(t) + wantQuery(t, u, "/search.json", map[string]string{"engine": "extract", "url": "https://example.com", "api_key": extractKey}) + wantQuery(t, u, "/search.json", tc.want) + }) + } +} + +func TestMapAndCrawlUsageErrorsAreUnsent(t *testing.T) { + for _, tc := range []struct { + name string + args []string + msg string + }{ + {"crawl with mode=map", []string{"crawl", "https://example.com", "mode=map"}, + "crawl always uses mode=crawl; drop mode=map, or use 'serpapi extract URL mode=map'"}, + {"map with mode=scrape", []string{"map", "https://example.com", "mode=scrape"}, "map always uses mode=map; drop mode=scrape"}, + {"map without a URL", []string{"map"}, "No URL given, e.g. serpapi map https://example.com"}, + {"map with two URLs", []string{"map", "https://a.example", "https://b.example"}, + "map takes one URL, got 2: https://a.example, https://b.example; to extract several pages of one website, use 'serpapi crawl URL'"}, + {"crawl without a scheme", []string{"crawl", "example.com"}, "e.g. https://example.com"}, + {"search's --max-pages on crawl", []string{"crawl", "https://example.com", "--max-pages", "5"}, + "unknown flag: --max-pages; API parameters are key=value pairs, e.g. max_pages=VALUE"}, + {"search's --all-pages on crawl", []string{"crawl", "https://example.com", "--all-pages"}, + "unknown flag: --all-pages; --all-pages works only with search; API parameters are key=value pairs, e.g. max_pages=5 max_depth=1"}, + {"search's --image on map", []string{"map", "https://example.com", "--image", "photo.png"}, + "unknown flag: --image; --image works only with search; API parameters are key=value pairs, e.g. include_sitemap=false"}, + {"crawl parameter without =", []string{"crawl", "https://example.com", "max_pages", "5"}, + `Unexpected argument "max_pages": API parameters are key=value pairs, e.g. max_pages=5 max_depth=1`}, + {"one-letter flag on map", []string{"map", "https://example.com", "-s"}, "e.g. include_sitemap=false"}, + } { + t.Run(tc.name, func(t *testing.T) { + fakeAPI(t, nil) + res := execute(t, "", append([]string{"--api-key", extractKey}, tc.args...)...) + wantError(t, res, 2, "usage_error", tc.msg) + }) + } +} + +// A crawl given two URLs is not told to use crawl. +func TestCrawlWithTwoURLsGetsNoCrawlAdvice(t *testing.T) { + fakeAPI(t, nil) + + res := execute(t, "", "--api-key", extractKey, "crawl", "https://a.example", "https://b.example") + + if msg, _ := wantError(t, res, 2, "usage_error", "crawl takes one URL, got 2")["message"].(string); strings.Contains(msg, "serpapi crawl") { + t.Errorf("message = %q, want no advice to use crawl", msg) + } +} + +func TestMapAndCrawlNeedAKeyBeforeSending(t *testing.T) { + for _, name := range []string{"map", "crawl"} { + t.Run(name, func(t *testing.T) { + fakeAPI(t, nil) + wantError(t, execute(t, "", name, "https://example.com"), 2, "usage_error", "No API key") + }) + } +} + +// --debug and --timeout are global flags; extract, map and crawl must honor +// them as search does: a JSON trace on piped stderr, without the key, and the +// timeout in effect. +func TestExtractCommandsTraceWithDebug(t *testing.T) { + for _, name := range []string{"extract", "map", "crawl"} { + t.Run(name, func(t *testing.T) { + fakeAPI(t, respond(debugSearchBody)) + + res := execute(t, "", "--api-key", extractKey, "--debug", "--timeout", "120", name, "https://example.com/") + + var events []string + for _, line := range strings.Split(strings.TrimSpace(res.stderr), "\n") { + var v struct { + Debug *struct { + Event string `json:"event"` + } `json:"debug"` + } + if err := json.Unmarshal([]byte(line), &v); err != nil || v.Debug == nil { + t.Fatalf("stderr line is not a JSON debug object: %q", line) + } + events = append(events, v.Debug.Event) + } + if res.code != 0 { + t.Fatalf("exit code = %d, want 0; stderr: %s", res.code, res.stderr) + } + joined := strings.Join(events, "\n") + for _, want := range []string{"GET http", "api_key=[REDACTED]", "engine=extract", "Timeout: 120s", "Server search: id=abc123", "Server timeline:"} { + if !strings.Contains(joined, want) { + t.Errorf("trace missing %q:\n%s", want, joined) + } + } + if strings.Contains(res.stderr, extractKey) { + t.Error("trace leaked the API key") + } + if !strings.Contains(res.stdout, `"abc123"`) { + t.Errorf("stdout = %q, want the result untouched by the trace", res.stdout) + } + }) + } +} + +func TestExtractCommandsTimeOut(t *testing.T) { + const extraction = "Request to /search.json timed out after 50ms: retry with a higher --timeout, such as --timeout 120" + const crawl = "Request to /search.json timed out after 50ms. A crawl extracts every page it visits, so it takes longer than one page: " + + "retry with a higher --timeout, such as --timeout 300" + for _, tc := range []struct { + name, env, want string + args []string + }{ + {"extract --timeout", "", extraction, []string{"--timeout", "0.05", "extract"}}, + {"map --timeout", "", extraction, []string{"--timeout", "50ms", "map"}}, + {"extract mode=crawl", "", crawl, []string{"--timeout", "50ms", "extract", "mode=crawl"}}, + {"crawl --timeout", "", crawl, []string{"--timeout", "0.05", "crawl"}}, + {"crawl SERPAPI_TIMEOUT", "0.05", crawl, []string{"crawl"}}, + } { + t.Run(tc.name, func(t *testing.T) { + fakeAPI(t, hang) + t.Setenv("SERPAPI_TIMEOUT", tc.env) + + start := time.Now() + res := execute(t, "", append(append([]string{"--api-key", extractKey}, tc.args...), "https://example.com/")...) + + wantError(t, res, 1, "network_error", tc.want) + if strings.Contains(res.stdout+res.stderr, "Slow engines") { + t.Errorf("advice blames slow engines:\n%s%s", res.stdout, res.stderr) + } + if elapsed := time.Since(start); elapsed > 2*time.Second { + t.Errorf("took %v, want the 50ms timeout to stop it", elapsed) + } + }) + } +} + +func TestIsURLArg(t *testing.T) { + for arg, want := range map[string]bool{ + "https://example.com/?a=1&b=2": true, + "example.com": true, + "example.com?a=1": true, + "example.com/path?a=1": true, + "": true, + "max_pages": false, + "localhost": false, + "mode=map": false, + "url=https://example.com/?a=1": false, + "include_tags=article, main": false, + "exclude_tags=.ads,#cookie": false, + } { + if got := isURLArg(arg); got != want { + t.Errorf("isURLArg(%q) = %v, want %v", arg, got, want) + } + } +} diff --git a/pkg/cmd/root.go b/pkg/cmd/root.go index e6b6871..772a480 100644 --- a/pkg/cmd/root.go +++ b/pkg/cmd/root.go @@ -69,7 +69,7 @@ Piped, it is indented too, so head, grep and file viewers show whole lines; head cuts the JSON short, so take parts with --jq. A terminal gets control characters from results escaped, so a page cannot drive it; piped output keeps them. Keep results small: - --fields "organic_results[].{title, link}" server-side filter (search only) + --fields "organic_results[].{title, link}" server-side filter (search, extract, map, crawl) --jq ".organic_results[].link" jq filter; strings print unquoted output=md Markdown instead of JSON @@ -98,10 +98,12 @@ hour old, so add it only when that is too old. zero_trace=true (Enterprise) stores nothing, so it skips the cache too and leaves no search for archive to read. account, locations and archive are free, and so is an image upload, though it needs searches left this month and this hour, and has its own hourly -limit.`, +limit. crawl counts one search per page it extracts.`, Example: ` serpapi search engine=google q=coffee --fields "organic_results[0:3]" serpapi search engine=google coffee shops --jq ".organic_results[].link" serpapi search engine=google q=coffee output=md + serpapi extract https://example.com output=md + serpapi crawl https://example.com/docs/ max_pages=5 --jq ".pages[].url" serpapi locations q=Austin limit=3 serpapi account --jq .total_searches_left`, Version: version.Version, @@ -237,7 +239,7 @@ func runHelp(c *cobra.Command, args []string) error { func init() { pf := rootCmd.PersistentFlags() pf.StringVar(&apiKeyFlag, "api-key", "", "SerpApi API key; prefer SERPAPI_KEY, since flags show up in ps and shell history") - pf.StringVar(&fieldsFlag, "fields", "", `search only: server-side json_restrictor, e.g. "organic_results[].{title, link}"`) + pf.StringVar(&fieldsFlag, "fields", "", `search, extract, map and crawl only: server-side json_restrictor, e.g. "organic_results[].{title, link}"`) pf.StringVar(&jqFlag, "jq", "", `jq filter applied to the JSON result, e.g. ".organic_results[].link"`) pf.StringVar(&timeoutFlag, "timeout", "", fmt.Sprintf( "HTTP request timeout in seconds, such as 120, or a duration, such as 2m; 0 waits indefinitely (default %d; env SERPAPI_TIMEOUT)", @@ -257,22 +259,35 @@ func init() { // starting with "-", such as -site:example.com, and an API parameter guessed // as a flag, such as --num 10. func flagErrorHint(cmd *cobra.Command, msg string) string { - if cmd != searchCmd && cmd != locationsCmd { - return msg - } - params := "API parameters are key=value pairs, e.g. engine=google q=coffee" - if cmd == locationsCmd { + var params string + switch cmd { + case searchCmd: + params = "API parameters are key=value pairs, e.g. engine=google q=coffee" + case locationsCmd: params = "API parameters are key=value pairs, e.g. q=Austin limit=3" + case extractCmd, mapCmd, crawlCmd: + params = "API parameters are key=value pairs, e.g. " + extractParamExample(cmd) + default: + return msg } if strings.HasPrefix(msg, "unknown shorthand flag") { // pflag reports "unknown shorthand flag: 'e' in -e". A one-letter flag // is a guessed parameter; a longer one may be a word like -site:x.com. - if _, arg, _ := strings.Cut(msg, " in "); len(arg) == 2 { + if _, arg, _ := strings.Cut(msg, " in "); len(arg) == 2 || isExtractCommand(cmd) { return msg + "; " + params } return msg + `; to search for a word starting with "-", put it inside q="..." or after --; ` + params } if name, ok := strings.CutPrefix(msg, "unknown flag: --"); ok { + if isExtractCommand(cmd) { + // search's --all-pages and --image have no Extract API parameter; + // its --max-pages does, as crawl's max_pages. + if name != "max-pages" && searchCmd.Flags().Lookup(name) != nil { + return msg + "; --" + name + " works only with search; " + params + } + // Extract API parameters use underscores, as in max_pages. + name = strings.ReplaceAll(name, "-", "_") + } return msg + "; API parameters are key=value pairs, e.g. " + name + "=VALUE" } return msg @@ -287,8 +302,8 @@ func validateGlobalFlags(cmd *cobra.Command, _ []string) error { } requestTimeout = timeout debugRequests = resolveDebug() - if fieldsFlag != "" && cmd != searchCmd { - return &clierrors.UsageError{Message: "--fields works only with search; use --jq to filter " + cmd.Name() + " output"} + if fieldsFlag != "" && cmd != searchCmd && !isExtractCommand(cmd) { + return &clierrors.UsageError{Message: "--fields works only with search, extract, map and crawl; use --jq to filter " + cmd.Name() + " output"} } if jqFlag != "" { if _, err := jq.Compile(jqFlag); err != nil { diff --git a/tests/e2e_test.go b/tests/e2e_test.go index ad0eae1..2214111 100644 --- a/tests/e2e_test.go +++ b/tests/e2e_test.go @@ -125,6 +125,10 @@ func TestUsageErrorsWithoutNetwork(t *testing.T) { {[]string{"--api-key", "x", "archive", "../bad"}, "Invalid archive ID"}, {[]string{"--api-key", "x", "search", "engine=google", "q=coffee", "output=xml"}, "Unknown output=xml"}, {[]string{"locations"}, "Pass a location query"}, + {[]string{"extract", "https://example.com"}, "No API key found"}, + {[]string{"--api-key", "x", "extract", "example.com"}, "e.g. https://example.com"}, + {[]string{"crawl", "https://example.com", "mode=map"}, "crawl always uses mode=crawl"}, + {[]string{"map", "https://example.com"}, "No API key found"}, } for _, tt := range tests { res := serpapi(t, "", tt.args...) @@ -279,6 +283,38 @@ func TestArchive(t *testing.T) { } } +func TestExtract(t *testing.T) { + key := requireLive(t) + res := serpapi(t, "", "--api-key", key, "--jq", ".page_information.title", "extract", "https://example.com/") + wantSuccess(t, res) + if got := strings.TrimSpace(res.stdout); got != "Example Domain" { + t.Errorf("page title = %q, want Example Domain", got) + } + + res = serpapi(t, "", "--api-key", key, "extract", "https://example.com/", "output=md") + wantSuccess(t, res) + // Markdown opens with the search's metadata as YAML front matter. + if !strings.HasPrefix(res.stdout, "---\n") || !strings.Contains(res.stdout, "engine: extract") || !strings.Contains(res.stdout, "documentation examples") { + t.Errorf("want the page as Markdown, got %q", res.stdout) + } +} + +// TestMapAndCrawl costs two searches: a map, and a crawl of one page. +func TestMapAndCrawl(t *testing.T) { + key := requireLive(t) + res := serpapi(t, "", "--api-key", key, "--jq", ".search_parameters.mode", "map", "https://example.com/") + wantSuccess(t, res) + if got := strings.TrimSpace(res.stdout); got != "map" { + t.Errorf("map ran mode %q, want map", got) + } + + res = serpapi(t, "", "--api-key", key, "--jq", ".pages | map(.url) | join(\" \")", "crawl", "https://example.com/", "max_pages=1") + wantSuccess(t, res) + if got := strings.TrimSpace(res.stdout); got != "https://example.com/" { + t.Errorf("crawl pages = %q, want only https://example.com/", got) + } +} + // smallest valid 1x1 transparent PNG var png1x1 = []byte{ 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52, From 3a994079bafbd30eb90897b11e55cec81e68c00c Mon Sep 17 00:00:00 2001 From: Josef Strzibny Date: Mon, 26 Oct 2026 09:37:04 +0100 Subject: [PATCH 2/2] [Extract] Wait up to 300s for a crawl and fix flag and URL hints A crawl extracts up to max_pages pages in one request, and SerpApi still finishes one the CLI gave up on and charges it one search per page. Without --timeout or SERPAPI_TIMEOUT, a crawl now waits up to 300s instead of 60s. api.DefaultCrawlTimeout names the limit, and the timeout advice uses it too: a crawl that times out is told to retry with --timeout 300, or twice the limit when that is more. The help and README give the new default. extract, map and crawl now send through search's sendSearch, which takes the spinner text, and search, extract and archive share checkOutput for the output= and --jq check. An empty URL, as from an unset variable, is reported before the URLs are counted, and mailto:me@x.com is no longer told to try https://mailto:me@x.com. search and locations now also suggest a dashed flag's parameter with underscores, so search --no-cache suggests no_cache=VALUE, not no-cache=VALUE. --max-pages suggests max_pages only on crawl; extract and map say it works only with search. The help says output=md puts the search metadata before the Markdown, and that only search joins words without "=" into q. The README says a crawl counts one search per page and mentions include_subdomains=true. Each behavior is tested once, and one table checks the timeout each command runs with. Co-Authored-By: Claude Opus 5.5 --- README.md | 10 +-- pkg/api/client.go | 6 +- pkg/cmd/archive.go | 5 +- pkg/cmd/edge_test.go | 2 + pkg/cmd/extract.go | 50 ++++++-------- pkg/cmd/extract_test.go | 149 +++++++++++++++++++++++----------------- pkg/cmd/root.go | 48 ++++++------- pkg/cmd/root_test.go | 17 ++--- pkg/cmd/search.go | 24 ++++--- tests/e2e_test.go | 1 - 10 files changed, 168 insertions(+), 144 deletions(-) diff --git a/README.md b/README.md index 665e615..af73083 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ serpapi extract https://example.com output=md serpapi extract https://example.com --jq '.inline_links[].link' ``` -It counts one search. The Extract API runs no JavaScript and follows each site's `robots.txt`. Search pages that have their own engine, such as Google or Bing, are redirected to it and return that engine's JSON. Unlike `search`, `extract`, `map` and `crawl` need an API key. +Scraping a page and mapping a website count one search each; a crawl counts one per page it extracts. The Extract API runs no JavaScript and follows each site's `robots.txt`. Search pages that have their own engine, such as Google or Bing, are redirected to it and return that engine's JSON. Unlike `search`, `extract`, `map` and `crawl` need an API key. ### map @@ -119,14 +119,14 @@ serpapi map https://books.toscrape.com --jq '.urls[].url' `serpapi crawl URL` is a shortcut for `serpapi extract URL mode=crawl`: it sends the same request and takes the same parameters and flags. -Starts at the URL, follows links on the same website and extracts every page it visits, one entry each in `pages`. `max_pages` caps the pages, the first included (default 10, at most 100), and `max_depth` how many links deep it goes (default 2, at most 3; `0` is only the first page). +Starts at the URL, follows links on the same website and extracts every page it visits, one entry each in `pages`. `max_pages` caps the pages, the first included (default 10, at most 100), and `max_depth` how many links deep it goes (default 2, at most 3; `0` is only the first page). `include_subdomains=true` also follows links to other subdomains. ```bash serpapi crawl https://books.toscrape.com max_pages=5 --jq '.pages[].url' -serpapi crawl https://books.toscrape.com max_pages=20 max_depth=1 --timeout 300 +serpapi crawl https://books.toscrape.com max_pages=20 max_depth=1 ``` -It counts one search per page it extracts, so the default `max_pages=10` can cost 10; cached and failed pages are free, and `search_metadata.credits_consumed` says what it cost. A crawl can outlast the default 60s `--timeout`, so raise it; `timeout=` is the API's own per-page fetch limit in milliseconds. +It counts one search per page it extracts, so the default `max_pages=10` can cost 10; cached and failed pages are free, and `search_metadata.credits_consumed` says what it cost. Without `--timeout` or `SERPAPI_TIMEOUT`, a crawl waits up to 300s instead of 60s, since one that times out still runs and is charged; `timeout=` is the API's own per-page fetch limit in milliseconds. ### archive @@ -188,7 +188,7 @@ Prints the CLI version, as `--version` does. - `--jq `: [jq](https://jqlang.github.io/jq/manual/) filter applied to the JSON result, like `gh --jq`; strings print unquoted and `null` as an empty line, and results stream as they are produced. A filter that prints only `null` gets a warning listing the response's top-level keys. jq is built in, so nothing else needs installing. In bash and zsh, quote the expression with single quotes. - `--fields `: search, extract, map and crawl only; see [search](#search). - `--api-key `: API key. Prefer `SERPAPI_KEY`, since flags show up in `ps` and shell history. -- `--timeout `: HTTP request timeout in seconds, such as `120`, or a duration, such as `2m` (default `60`; env `SERPAPI_TIMEOUT`). `0` waits indefinitely. Slow engines such as `google_ai_mode`, and searches with `no_cache=true`, can take longer than the default, as can `serpapi crawl`, which extracts every page it visits; raise it when a `network_error` says the request timed out, and add `--debug` to see which step stalled. A timed-out search may still finish on SerpApi's side, count against your searches and be available in your search archive. +- `--timeout `: HTTP request timeout in seconds, such as `120`, or a duration, such as `2m` (default `60`, `300` for a crawl; env `SERPAPI_TIMEOUT`). `0` waits indefinitely. Slow engines such as `google_ai_mode`, and searches with `no_cache=true`, can take longer than the default; raise it when a `network_error` says the request timed out, and add `--debug` to see which step stalled. A timed-out search may still finish on SerpApi's side, count against your searches and be available in your search archive. ```bash # Give an AI Mode query up to two minutes serpapi search --timeout 120 engine=google_ai_mode q="..." diff --git a/pkg/api/client.go b/pkg/api/client.go index 7edbc2a..04b9c57 100644 --- a/pkg/api/client.go +++ b/pkg/api/client.go @@ -31,6 +31,10 @@ const ( // Slow engines such as google_ai_mode can take longer with no_cache=true, // and SerpApi still finishes and counts a search the client gave up on. DefaultTimeout = 60 * time.Second + // DefaultCrawlTimeout bounds an Extract API crawl, which extracts up to + // max_pages pages in one request. SerpApi still finishes a crawl the + // client gave up on and charges it one search per page. + DefaultCrawlTimeout = 300 * time.Second ) // maxResponseBytes caps the response body; a variable so tests can lower it. @@ -224,7 +228,7 @@ func (c *Client) networkError(endpoint string, query url.Values, err error) erro message = timedOut + ": " + retry(suggestedTimeout(c.http.Timeout)) case strings.EqualFold(query.Get("engine"), "extract") && strings.EqualFold(query.Get("mode"), "crawl"): message = timedOut + ". A crawl extracts every page it visits, so it takes longer than one page: " + - retry(max(suggestedTimeout(c.http.Timeout), 300)) + + retry(max(suggestedTimeout(c.http.Timeout), int64(DefaultCrawlTimeout/time.Second))) + ". The crawl may still finish on SerpApi's side, count one search per page and be available in your search archive" case strings.EqualFold(query.Get("engine"), "extract"): message = timedOut + ": " + retry(suggestedTimeout(c.http.Timeout)) + ". The extraction" + onSerpApi diff --git a/pkg/cmd/archive.go b/pkg/cmd/archive.go index 9cbcb08..d3efde9 100644 --- a/pkg/cmd/archive.go +++ b/pkg/cmd/archive.go @@ -44,13 +44,10 @@ func runArchive(cmd *cobra.Command, args []string) error { } format = v } - raw, err := isRawOutput(format) + raw, err := checkOutput(format) if err != nil { return err } - if raw && jqFlag != "" { - return &clierrors.UsageError{Message: "--jq needs JSON, but output=" + format + " is not JSON"} - } apiKey, err := resolveAPIKey() if err != nil { diff --git a/pkg/cmd/edge_test.go b/pkg/cmd/edge_test.go index 8c29590..7b0690f 100644 --- a/pkg/cmd/edge_test.go +++ b/pkg/cmd/edge_test.go @@ -104,6 +104,8 @@ func edgeCases() []edgeCase { {name: "flags/unknown flag", args: searchWith("--bogus"), exit: 2, code: "usage_error", msg: "unknown flag: --bogus; API parameters are key=value pairs, e.g. bogus=VALUE"}, {name: "flags/API parameter as a flag", args: searchWith("--num", "10"), exit: 2, code: "usage_error", msg: "e.g. num=VALUE"}, + {name: "flags/dashed API parameter as a flag", args: searchWith("--no-cache"), exit: 2, code: "usage_error", + msg: "unknown flag: --no-cache; API parameters are key=value pairs, e.g. no_cache=VALUE", exact: true}, {name: "flags/one-letter flag guess", args: []string{"search", "-e", "youtube", "--search_query", "Rick"}, exit: 2, code: "usage_error", msg: "unknown shorthand flag: 'e' in -e; API parameters are key=value pairs, e.g. engine=google q=coffee", exact: true}, {name: "flags/unknown flag outside search", args: []string{"account", "--bogus"}, exit: 2, code: "usage_error", msg: "unknown flag: --bogus", exact: true}, diff --git a/pkg/cmd/extract.go b/pkg/cmd/extract.go index e9e1b48..e5ac900 100644 --- a/pkg/cmd/extract.go +++ b/pkg/cmd/extract.go @@ -1,15 +1,15 @@ package cmd import ( - "encoding/json" "fmt" "net/url" + "slices" "strings" "github.com/spf13/cobra" + "github.com/serpapi/serpapi-cli/pkg/api" clierrors "github.com/serpapi/serpapi-cli/pkg/errors" - "github.com/serpapi/serpapi-cli/pkg/output" "github.com/serpapi/serpapi-cli/pkg/params" ) @@ -26,7 +26,8 @@ also be given as url=URL. Other parameters are key=value pairs sent as-is: mode=crawl every page of the website it visits; same as 'serpapi crawl' include_tags=, exclude_tags= CSS selectors to keep or drop include_markdown=true add the main content as Markdown to JSON - output=md only the main content, as Markdown + output=md the main content as Markdown, after the + search metadata Search pages with their own engine, such as Google or Bing, are redirected to it and return that engine's JSON. The Extract API runs no JavaScript and obeys robots.txt, so a page that disallows it returns an error. @@ -84,11 +85,11 @@ per page in pages. Cost: one search per page extracted, the first included, so the default max_pages=10 can cost 10; cached and failed pages are free. -search_metadata.credits_consumed says what it cost. A crawl can outlast the -60s --timeout: raise it, e.g. --timeout 300. timeout= is the Extract API's own -per-page fetch limit in milliseconds.`, +search_metadata.credits_consumed says what it cost. Without --timeout or +SERPAPI_TIMEOUT, a crawl waits up to 300s, not 60s. timeout= is the Extract +API's own per-page fetch limit in milliseconds.`, Example: ` serpapi crawl https://books.toscrape.com max_pages=5 --jq ".pages[].url" - serpapi crawl https://books.toscrape.com max_pages=20 max_depth=1 --timeout 300 + serpapi crawl https://books.toscrape.com max_pages=20 max_depth=1 serpapi crawl https://example.com/docs/ --fields "pages[].{url, text_content}"`, Args: cobra.ArbitraryArgs, RunE: runExtractMode("crawl"), @@ -116,13 +117,10 @@ func runExtract(cmd *cobra.Command, args []string, mode string) error { } raw := false if o, ok := p["output"]; ok { - if raw, err = isRawOutput(o); err != nil { + if raw, err = checkOutput(o); err != nil { return err } } - if raw && jqFlag != "" { - return &clierrors.UsageError{Message: "--jq needs JSON, but output=" + p["output"] + " is not JSON"} - } restricted := params.ApplyFields(p, fieldsFlag, "error") != "" // Unlike search, the Extract API answers nothing without a key. @@ -130,25 +128,12 @@ func runExtract(cmd *cobra.Command, args []string, mode string) error { if err != nil { return err } - client := newClient(apiKey) - ctx := cmd.Context() - label := extractLabel(p["mode"]) - - if raw { - body, err := withSpinner(label, func() ([]byte, error) { return client.SearchRaw(ctx, p) }) - if err != nil { - return err - } - return outputError(output.PrintRaw(cmd.OutOrStdout(), body)) - } - result, err := withSpinner(label, func() (json.RawMessage, error) { return client.Search(ctx, p) }) - if err != nil { - return err - } - if restricted && isEmptyObject(result) { - warnNothingMatched(cmd) + // A crawl extracts up to max_pages pages in one request, and one the CLI + // gave up on still runs and is charged, so it gets longer by default. + if strings.EqualFold(p["mode"], "crawl") && !timeoutSet { + requestTimeout = api.DefaultCrawlTimeout } - return handleOutput(cmd, result, searchHint(p)) + return sendSearch(cmd, newClient(apiKey), p, extractLabel(p["mode"]), raw, restricted) } // extractParams turns the arguments into the parameters of an Extract API @@ -182,6 +167,10 @@ func extractParams(cmd *cobra.Command, args []string, mode string) (map[string]s p := params.ParamsToMap(parsed) name := cmd.Name() + // An empty URL, as from an unset "$URL", would be a blank in the list below. + if slices.Contains(urls, "") { + return nil, checkExtractURL("") + } switch len(urls) { case 0: return nil, &clierrors.UsageError{Message: "No URL given, e.g. serpapi " + name + " https://example.com"} @@ -241,7 +230,8 @@ func checkExtractURL(raw string) error { } msg := fmt.Sprintf("Invalid URL %q: the Extract API takes an absolute http or https URL", raw) if !strings.Contains(raw, "://") { - if u, err := url.Parse("https://" + raw); err == nil && strings.Contains(u.Hostname(), ".") { + // Not mailto:me@x.com, which would parse as user "mailto:me" on x.com. + if u, err := url.Parse("https://" + raw); err == nil && u.User == nil && strings.Contains(u.Hostname(), ".") { msg += ", e.g. https://" + raw } } diff --git a/pkg/cmd/extract_test.go b/pkg/cmd/extract_test.go index b9a21e9..27ed69f 100644 --- a/pkg/cmd/extract_test.go +++ b/pkg/cmd/extract_test.go @@ -89,6 +89,7 @@ func TestExtractUsageErrorsAreUnsent(t *testing.T) { {"two URLs", []string{"https://a.example", "https://b.example"}, "extract takes one URL, got 2: https://a.example, https://b.example"}, {"URL and url=", []string{"https://a.example", "url=https://b.example"}, "extract takes one URL, got 2"}, {"two url=", []string{"url=https://a.example", "url=https://b.example"}, "extract takes one URL, got 2: https://a.example, https://b.example"}, + {"empty URL with a URL", []string{"", "https://example.com"}, `Invalid URL "": the Extract API takes an absolute http or https URL`}, {"parameter without =", []string{"https://example.com", "main"}, `Unexpected argument "main": API parameters are key=value pairs, e.g. include_tags=main`}, {"no scheme", []string{"example.com/page"}, `Invalid URL "example.com/page": the Extract API takes an absolute http or https URL, e.g. https://example.com/page`}, @@ -101,6 +102,8 @@ func TestExtractUsageErrorsAreUnsent(t *testing.T) { {"unknown output", []string{"https://example.com", "output=xml"}, "Unknown output=xml"}, {"parameter as a flag", []string{"https://example.com", "--mode", "map"}, "unknown flag: --mode; API parameters are key=value pairs, e.g. mode=VALUE"}, {"one-letter flag", []string{"https://example.com", "-m", "map"}, "API parameters are key=value pairs, e.g. include_tags=main"}, + {"search's --max-pages", []string{"https://example.com", "--max-pages", "3"}, + "unknown flag: --max-pages; --max-pages works only with search; API parameters are key=value pairs, e.g. include_tags=main"}, } { t.Run(tc.name, func(t *testing.T) { fakeAPI(t, nil) @@ -167,6 +170,8 @@ func TestMapAndCrawlUsageErrorsAreUnsent(t *testing.T) { {"crawl without a scheme", []string{"crawl", "example.com"}, "e.g. https://example.com"}, {"search's --max-pages on crawl", []string{"crawl", "https://example.com", "--max-pages", "5"}, "unknown flag: --max-pages; API parameters are key=value pairs, e.g. max_pages=VALUE"}, + {"search's --max-pages on map", []string{"map", "https://example.com", "--max-pages", "5"}, + "unknown flag: --max-pages; --max-pages works only with search; API parameters are key=value pairs, e.g. include_sitemap=false"}, {"search's --all-pages on crawl", []string{"crawl", "https://example.com", "--all-pages"}, "unknown flag: --all-pages; --all-pages works only with search; API parameters are key=value pairs, e.g. max_pages=5 max_depth=1"}, {"search's --image on map", []string{"map", "https://example.com", "--image", "photo.png"}, @@ -194,88 +199,106 @@ func TestCrawlWithTwoURLsGetsNoCrawlAdvice(t *testing.T) { } } -func TestMapAndCrawlNeedAKeyBeforeSending(t *testing.T) { - for _, name := range []string{"map", "crawl"} { - t.Run(name, func(t *testing.T) { - fakeAPI(t, nil) - wantError(t, execute(t, "", name, "https://example.com"), 2, "usage_error", "No API key") - }) - } -} +// --debug is a global flag; extract must honor it as search does: a JSON +// trace on piped stderr, without the key, and stdout untouched. +func TestExtractTracesWithDebug(t *testing.T) { + fakeAPI(t, respond(debugSearchBody)) -// --debug and --timeout are global flags; extract, map and crawl must honor -// them as search does: a JSON trace on piped stderr, without the key, and the -// timeout in effect. -func TestExtractCommandsTraceWithDebug(t *testing.T) { - for _, name := range []string{"extract", "map", "crawl"} { - t.Run(name, func(t *testing.T) { - fakeAPI(t, respond(debugSearchBody)) - - res := execute(t, "", "--api-key", extractKey, "--debug", "--timeout", "120", name, "https://example.com/") - - var events []string - for _, line := range strings.Split(strings.TrimSpace(res.stderr), "\n") { - var v struct { - Debug *struct { - Event string `json:"event"` - } `json:"debug"` - } - if err := json.Unmarshal([]byte(line), &v); err != nil || v.Debug == nil { - t.Fatalf("stderr line is not a JSON debug object: %q", line) - } - events = append(events, v.Debug.Event) - } - if res.code != 0 { - t.Fatalf("exit code = %d, want 0; stderr: %s", res.code, res.stderr) - } - joined := strings.Join(events, "\n") - for _, want := range []string{"GET http", "api_key=[REDACTED]", "engine=extract", "Timeout: 120s", "Server search: id=abc123", "Server timeline:"} { - if !strings.Contains(joined, want) { - t.Errorf("trace missing %q:\n%s", want, joined) - } - } - if strings.Contains(res.stderr, extractKey) { - t.Error("trace leaked the API key") - } - if !strings.Contains(res.stdout, `"abc123"`) { - t.Errorf("stdout = %q, want the result untouched by the trace", res.stdout) - } - }) + res := execute(t, "", "--api-key", extractKey, "--debug", "extract", "https://example.com/") + + var events []string + for _, line := range strings.Split(strings.TrimSpace(res.stderr), "\n") { + var v struct { + Debug *struct { + Event string `json:"event"` + } `json:"debug"` + } + if err := json.Unmarshal([]byte(line), &v); err != nil || v.Debug == nil { + t.Fatalf("stderr line is not a JSON debug object: %q", line) + } + events = append(events, v.Debug.Event) + } + if res.code != 0 { + t.Fatalf("exit code = %d, want 0; stderr: %s", res.code, res.stderr) + } + joined := strings.Join(events, "\n") + for _, want := range []string{"GET http", "api_key=[REDACTED]", "engine=extract", "Server search: id=abc123", "Server timeline:"} { + if !strings.Contains(joined, want) { + t.Errorf("trace missing %q:\n%s", want, joined) + } + } + if strings.Contains(res.stderr, extractKey) { + t.Error("trace leaked the API key") + } + if !strings.Contains(res.stdout, `"abc123"`) { + t.Errorf("stdout = %q, want the result untouched by the trace", res.stdout) } } -func TestExtractCommandsTimeOut(t *testing.T) { - const extraction = "Request to /search.json timed out after 50ms: retry with a higher --timeout, such as --timeout 120" - const crawl = "Request to /search.json timed out after 50ms. A crawl extracts every page it visits, so it takes longer than one page: " + - "retry with a higher --timeout, such as --timeout 300" +// A crawl waits up to 300s unless --timeout or SERPAPI_TIMEOUT says +// otherwise; the trace shows the limit each request ran with. +func TestExtractTimeouts(t *testing.T) { for _, tc := range []struct { name, env, want string args []string }{ - {"extract --timeout", "", extraction, []string{"--timeout", "0.05", "extract"}}, - {"map --timeout", "", extraction, []string{"--timeout", "50ms", "map"}}, - {"extract mode=crawl", "", crawl, []string{"--timeout", "50ms", "extract", "mode=crawl"}}, - {"crawl --timeout", "", crawl, []string{"--timeout", "0.05", "crawl"}}, - {"crawl SERPAPI_TIMEOUT", "0.05", crawl, []string{"crawl"}}, + {"extract", "", "Timeout: 60s", []string{"extract"}}, + {"map", "", "Timeout: 60s", []string{"map"}}, + {"crawl", "", "Timeout: 300s", []string{"crawl"}}, + {"extract mode=crawl", "", "Timeout: 300s", []string{"extract", "mode=Crawl"}}, + {"crawl --timeout", "", "Timeout: 30s", []string{"--timeout", "30", "crawl"}}, + {"crawl --timeout 0", "", "Timeout: disabled", []string{"--timeout", "0", "crawl"}}, + {"crawl SERPAPI_TIMEOUT", "45", "Timeout: 45s", []string{"crawl"}}, + {"map --timeout", "", "Timeout: 120s", []string{"--timeout", "2m", "map"}}, } { t.Run(tc.name, func(t *testing.T) { - fakeAPI(t, hang) + fakeAPI(t, respond(`{"search_metadata":{"id":"e1"}}`)) t.Setenv("SERPAPI_TIMEOUT", tc.env) - start := time.Now() - res := execute(t, "", append(append([]string{"--api-key", extractKey}, tc.args...), "https://example.com/")...) + res := execute(t, "", append(append([]string{"--api-key", extractKey, "--debug"}, tc.args...), "https://example.com/")...) - wantError(t, res, 1, "network_error", tc.want) - if strings.Contains(res.stdout+res.stderr, "Slow engines") { - t.Errorf("advice blames slow engines:\n%s%s", res.stdout, res.stderr) + if res.code != 0 { + t.Fatalf("exit code = %d, want 0; stderr: %s", res.code, res.stderr) } - if elapsed := time.Since(start); elapsed > 2*time.Second { - t.Errorf("took %v, want the 50ms timeout to stop it", elapsed) + if !strings.Contains(res.stderr, `"event":"`+tc.want+`"`) { + t.Errorf("trace has no %q:\n%s", tc.want, res.stderr) } }) } } +// The timeout stops the request; its advice is the Extract API's, worded in +// pkg/api, not search's slow engines. +func TestCrawlTimesOut(t *testing.T) { + fakeAPI(t, hang) + + start := time.Now() + res := execute(t, "", "--api-key", extractKey, "--timeout", "50ms", "crawl", "https://example.com/") + + wantError(t, res, 1, "network_error", "Request to /search.json timed out after 50ms. A crawl extracts every page it visits") + if strings.Contains(res.stderr, "Slow engines") { + t.Errorf("advice blames slow engines: %s", res.stderr) + } + if elapsed := time.Since(start); elapsed > 2*time.Second { + t.Errorf("took %v, want the 50ms timeout to stop it", elapsed) + } +} + +func TestCheckExtractURLSuggestsHTTPS(t *testing.T) { + const takes = ": the Extract API takes an absolute http or https URL" + for raw, want := range map[string]string{ + "example.com/page": `Invalid URL "example.com/page"` + takes + ", e.g. https://example.com/page", + "example.com:8080/page": `Invalid URL "example.com:8080/page"` + takes + ", e.g. https://example.com:8080/page", + "mailto:me@x.com": `Invalid URL "mailto:me@x.com"` + takes, + "ftp://example.com": `Invalid URL "ftp://example.com"` + takes, + "": `Invalid URL ""` + takes, + } { + if err := checkExtractURL(raw); err == nil || err.Error() != "Usage error: "+want { + t.Errorf("checkExtractURL(%q) = %v, want %s", raw, err, want) + } + } +} + func TestIsURLArg(t *testing.T) { for arg, want := range map[string]bool{ "https://example.com/?a=1&b=2": true, diff --git a/pkg/cmd/root.go b/pkg/cmd/root.go index 772a480..f83e820 100644 --- a/pkg/cmd/root.go +++ b/pkg/cmd/root.go @@ -36,8 +36,10 @@ var ( debugFlag bool // requestTimeout is --timeout, else SERPAPI_TIMEOUT, as validateGlobalFlags - // resolved it for this run. + // resolved it for this run; timeoutSet says whether either was given, so + // a crawl can wait longer than the default. requestTimeout = api.DefaultTimeout + timeoutSet bool // debugRequests is --debug, else SERPAPI_DEBUG, as validateGlobalFlags // resolved it for this run. @@ -57,8 +59,8 @@ var rootCmd = &cobra.Command{ Short: "Search engine results as JSON from SerpApi", Long: `Search engine results as JSON from SerpApi (https://serpapi.com). -Parameters are key=value pairs sent to the API as-is; words without "=" are -joined into q. Engines and their parameters: https://serpapi.com/search-engine-apis +Parameters are key=value pairs sent to the API as-is; search joins words +without "=" into q. Engines and their parameters: https://serpapi.com/search-engine-apis API key, first match wins: --api-key, SERPAPI_KEY, SERPAPI_API_KEY, then the key saved by 'serpapi login'. Without one, search still runs, but the API @@ -82,10 +84,10 @@ answer: 200 the search ran but reported a problem, such as no results; 400 invalid parameters; 401 invalid API key; 403 account not allowed; 429 rate limited or out of searches; 5xx SerpApi failed. No status means no usable answer: a network_error, such as a timeout, or an unreadable response. -Requests time out after 60s: --timeout or SERPAPI_TIMEOUT sets seconds, 0 -waits indefinitely. --debug or SERPAPI_DEBUG=1 traces each request on stderr -(DNS, connect, TLS, first byte, headers, server timeline): plain text on a -terminal, otherwise one {"debug":{...}} JSON object per line. +Requests time out after 60s, a crawl after 300s: --timeout or SERPAPI_TIMEOUT +sets seconds, 0 waits indefinitely. --debug or SERPAPI_DEBUG=1 traces each +request on stderr (DNS, connect, TLS, first byte, headers, server timeline): +plain text on a terminal, otherwise one {"debug":{...}} JSON object per line. Exit 130, code interrupted: Ctrl-C or SIGTERM stopped it before it finished, after printing what had arrived. @@ -242,8 +244,8 @@ func init() { pf.StringVar(&fieldsFlag, "fields", "", `search, extract, map and crawl only: server-side json_restrictor, e.g. "organic_results[].{title, link}"`) pf.StringVar(&jqFlag, "jq", "", `jq filter applied to the JSON result, e.g. ".organic_results[].link"`) pf.StringVar(&timeoutFlag, "timeout", "", fmt.Sprintf( - "HTTP request timeout in seconds, such as 120, or a duration, such as 2m; 0 waits indefinitely (default %d; env SERPAPI_TIMEOUT)", - int(api.DefaultTimeout/time.Second))) + "HTTP request timeout in seconds, such as 120, or a duration, such as 2m; 0 waits indefinitely (default %d, %d for a crawl; env SERPAPI_TIMEOUT)", + int(api.DefaultTimeout/time.Second), int(api.DefaultCrawlTimeout/time.Second))) pf.BoolVar(&debugFlag, "debug", false, "trace each request on stderr: DNS, connect, TLS, first byte, headers and the server timeline; JSON lines off a terminal (env SERPAPI_DEBUG=1)") @@ -279,16 +281,13 @@ func flagErrorHint(cmd *cobra.Command, msg string) string { return msg + `; to search for a word starting with "-", put it inside q="..." or after --; ` + params } if name, ok := strings.CutPrefix(msg, "unknown flag: --"); ok { - if isExtractCommand(cmd) { - // search's --all-pages and --image have no Extract API parameter; - // its --max-pages does, as crawl's max_pages. - if name != "max-pages" && searchCmd.Flags().Lookup(name) != nil { - return msg + "; --" + name + " works only with search; " + params - } - // Extract API parameters use underscores, as in max_pages. - name = strings.ReplaceAll(name, "-", "_") + // search's --all-pages and --image have no Extract API parameter, and + // its --max-pages has one only in crawl, max_pages. + if isExtractCommand(cmd) && searchCmd.Flags().Lookup(name) != nil && (name != "max-pages" || cmd != crawlCmd) { + return msg + "; --" + name + " works only with search; " + params } - return msg + "; API parameters are key=value pairs, e.g. " + name + "=VALUE" + // API parameters use underscores, as in no_cache and max_pages. + return msg + "; API parameters are key=value pairs, e.g. " + strings.ReplaceAll(name, "-", "_") + "=VALUE" } return msg } @@ -296,11 +295,11 @@ func flagErrorHint(cmd *cobra.Command, msg string) string { // validateGlobalFlags rejects bad flags before any request, so a typo never // costs a search. func validateGlobalFlags(cmd *cobra.Command, _ []string) error { - timeout, err := resolveTimeout() + timeout, set, err := resolveTimeout() if err != nil { return err } - requestTimeout = timeout + requestTimeout, timeoutSet = timeout, set debugRequests = resolveDebug() if fieldsFlag != "" && cmd != searchCmd && !isExtractCommand(cmd) { return &clierrors.UsageError{Message: "--fields works only with search, extract, map and crawl; use --jq to filter " + cmd.Name() + " output"} @@ -314,16 +313,17 @@ func validateGlobalFlags(cmd *cobra.Command, _ []string) error { } // resolveTimeout returns --timeout, else SERPAPI_TIMEOUT, else -// api.DefaultTimeout. 0 disables the limit. -func resolveTimeout() (time.Duration, error) { +// api.DefaultTimeout, and whether either was given. 0 disables the limit. +func resolveTimeout() (time.Duration, bool, error) { raw, source := strings.TrimSpace(timeoutFlag), "--timeout" if raw == "" { raw, source = strings.TrimSpace(os.Getenv("SERPAPI_TIMEOUT")), "SERPAPI_TIMEOUT" } if raw == "" { - return api.DefaultTimeout, nil + return api.DefaultTimeout, false, nil } - return parseTimeout(raw, source) + d, err := parseTimeout(raw, source) + return d, true, err } // parseTimeout reads seconds, such as 120 or 1.5, or a duration, such as 2m. diff --git a/pkg/cmd/root_test.go b/pkg/cmd/root_test.go index 6139a8e..e6a4182 100644 --- a/pkg/cmd/root_test.go +++ b/pkg/cmd/root_test.go @@ -233,14 +233,15 @@ func TestResolveTimeout(t *testing.T) { flag string env string want time.Duration + wantSet bool wantErr bool }{ {name: "default", want: api.DefaultTimeout}, - {name: "flag seconds", flag: "120", want: 120 * time.Second}, - {name: "flag fractional", flag: "1.5", want: 1500 * time.Millisecond}, - {name: "flag zero disables", flag: "0", want: 0}, - {name: "env fallback", env: "90", want: 90 * time.Second}, - {name: "flag overrides env", flag: "10", env: "90", want: 10 * time.Second}, + {name: "flag seconds", flag: "120", want: 120 * time.Second, wantSet: true}, + {name: "flag fractional", flag: "1.5", want: 1500 * time.Millisecond, wantSet: true}, + {name: "flag zero disables", flag: "0", want: 0, wantSet: true}, + {name: "env fallback", env: "90", want: 90 * time.Second, wantSet: true}, + {name: "flag overrides env", flag: "10", env: "90", want: 10 * time.Second, wantSet: true}, {name: "negative rejected", flag: "-5", wantErr: true}, {name: "garbage rejected", flag: "soon", wantErr: true}, {name: "env garbage rejected", env: "abc", wantErr: true}, @@ -252,7 +253,7 @@ func TestResolveTimeout(t *testing.T) { t.Cleanup(func() { timeoutFlag = "" }) t.Setenv("SERPAPI_TIMEOUT", tt.env) - got, err := resolveTimeout() + got, set, err := resolveTimeout() if tt.wantErr { var ue *clierrors.UsageError if !errors.As(err, &ue) { @@ -263,8 +264,8 @@ func TestResolveTimeout(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - if got != tt.want { - t.Errorf("expected %s, got %s", tt.want, got) + if got != tt.want || set != tt.wantSet { + t.Errorf("expected %s (set %v), got %s (set %v)", tt.want, tt.wantSet, got, set) } }) } diff --git a/pkg/cmd/search.go b/pkg/cmd/search.go index 1a9a431..876e78c 100644 --- a/pkg/cmd/search.go +++ b/pkg/cmd/search.go @@ -129,13 +129,10 @@ func runSearch(cmd *cobra.Command, args []string) error { } raw := false if o, ok := p["output"]; ok { - if raw, err = isRawOutput(o); err != nil { + if raw, err = checkOutput(o); err != nil { return err } } - if raw && jqFlag != "" { - return &clierrors.UsageError{Message: "--jq needs JSON, but output=" + p["output"] + " is not JSON"} - } if raw && allPagesFlag { return &clierrors.UsageError{Message: "--all-pages needs JSON, but output=" + p["output"] + " is not JSON"} } @@ -170,17 +167,19 @@ func runSearch(cmd *cobra.Command, args []string) error { if keyErr != config.ErrNoAPIKey && errors.As(keyErr, &cfgErr) { warn(cmd, "%s; searching without a key", cfgErr.Message) } - err = sendSearch(cmd, newClient(apiKey), p, raw, restrictor != "") + err = sendSearch(cmd, newClient(apiKey), p, "Searching...", raw, restrictor != "") if apiKey == "" { err = explainMissingKey(err) } return err } -func sendSearch(cmd *cobra.Command, client *api.Client, p map[string]string, raw, restricted bool) error { +// sendSearch sends p to /search.json and prints the answer, with label on the +// spinner. +func sendSearch(cmd *cobra.Command, client *api.Client, p map[string]string, label string, raw, restricted bool) error { ctx := cmd.Context() if raw { - body, err := withSpinner("Searching...", func() ([]byte, error) { return client.SearchRaw(ctx, p) }) + body, err := withSpinner(label, func() ([]byte, error) { return client.SearchRaw(ctx, p) }) if err != nil { return err } @@ -190,7 +189,7 @@ func sendSearch(cmd *cobra.Command, client *api.Client, p map[string]string, raw return searchAllPages(cmd, client, p, restricted) } - result, err := withSpinner("Searching...", func() (json.RawMessage, error) { return client.Search(ctx, p) }) + result, err := withSpinner(label, func() (json.RawMessage, error) { return client.Search(ctx, p) }) if err != nil { return err } @@ -214,6 +213,15 @@ func explainMissingKey(err error) error { } } +// checkOutput refuses an unknown output=format, or one that isn't JSON +// together with --jq, before any request, and reports whether it isn't JSON. +func checkOutput(format string) (raw bool, err error) { + if raw, err = isRawOutput(format); err == nil && raw && jqFlag != "" { + err = &clierrors.UsageError{Message: "--jq needs JSON, but output=" + format + " is not JSON"} + } + return raw, err +} + func isRawOutput(o string) (bool, error) { // The API lowercases output, so MD works like md. switch lower := strings.ToLower(o); { diff --git a/tests/e2e_test.go b/tests/e2e_test.go index 2214111..d3dea82 100644 --- a/tests/e2e_test.go +++ b/tests/e2e_test.go @@ -128,7 +128,6 @@ func TestUsageErrorsWithoutNetwork(t *testing.T) { {[]string{"extract", "https://example.com"}, "No API key found"}, {[]string{"--api-key", "x", "extract", "example.com"}, "e.g. https://example.com"}, {[]string{"crawl", "https://example.com", "mode=map"}, "crawl always uses mode=crawl"}, - {[]string{"map", "https://example.com"}, "No API key found"}, } for _, tt := range tests { res := serpapi(t, "", tt.args...)