From b7ce971630ddb4776a3773bb1863be8078c99224 Mon Sep 17 00:00:00 2001 From: Ryan Fowler Date: Mon, 13 Jul 2026 11:24:45 -0400 Subject: [PATCH] feat: add article Markdown output Extract readable HTML with Legible, convert it with htmd, and emit YAML frontmatter. Pass existing Markdown through with URL-only frontmatter and preserve existing output controls. --- AGENTS.md | 1 + Cargo.lock | 403 +++++++++++++++++++++++++++++++++++- Cargo.toml | 2 + README.md | 4 + docs/cli-reference.md | 27 +++ docs/output-formatting.md | 25 +++ src/cli.rs | 7 + src/cli/completion.rs | 6 + src/flag_registry.rs | 1 + src/format/article.rs | 129 ++++++++++++ src/format/mod.rs | 1 + src/http/mod.rs | 10 +- src/http/response.rs | 134 +++++++++++- src/http/response/stream.rs | 31 ++- tests/formatting.rs | 101 +++++++++ 15 files changed, 872 insertions(+), 10 deletions(-) create mode 100644 src/format/article.rs diff --git a/AGENTS.md b/AGENTS.md index eff28d99..a4b6194b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -105,6 +105,7 @@ Request flow: CLI parse → config merge → request build (gRPC may load/reflec - Pager: `--pager auto|on|off`; `NO_PAGER` disables auto fallback; `$PAGER` is shell-split but launched directly; `$LESS` suppresses fallback flags. Images/output files bypass pager. - `--copy` tees decoded stdout/output-file bodies to platform clipboard commands, skips >1 MiB, and bounds stdin/write/wait, killing hung backends with a warning. - Content type policy belongs in `src/format/content_type.rs`; update README/docs when user-visible formats or MIME behavior change. +- `--article` is an explicit buffered HTML/Markdown transformation: Legible extracts readable HTML, htmd converts it to Markdown, and existing Markdown passes through directly. `src/format/article.rs` adds scalar YAML frontmatter (all available article details for HTML, only the final URL for Markdown). It uses the final response URL for relative HTML links, caps decoded responses at 16 MiB and HTML DOM elements at 500,000, writes raw Markdown to files/clipboard, and only applies terminal Markdown styling afterward. ### gRPC/protobuf diff --git a/Cargo.lock b/Cargo.lock index d2f30f75..16f49ba2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -32,6 +32,12 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "anstream" version = "1.0.0" @@ -195,6 +201,21 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec 0.8.0", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bit-vec" version = "0.9.1" @@ -495,6 +516,29 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "cssparser" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dae61cf9c0abb83bd659dab65b7e4e38d8236824c85f0f804f173567bda257d2" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "phf", + "smallvec", +] + +[[package]] +name = "cssparser-macros" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "ctutils" version = "0.4.2" @@ -530,6 +574,27 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "syn", +] + [[package]] name = "difflib" version = "0.4.0" @@ -568,6 +633,36 @@ dependencies = [ "litrs", ] +[[package]] +name = "dom_query" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d9c2e7f1d22d0f2ce07626d259b8a55f4a47cb0938d4006dd8ae037f17d585e" +dependencies = [ + "bit-set", + "cssparser", + "foldhash 0.2.0", + "html5ever", + "precomputed-hash", + "selectors", + "tendril", +] + +[[package]] +name = "dtoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" + +[[package]] +name = "dtoa-short" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" +dependencies = [ + "dtoa", +] + [[package]] name = "dunce" version = "1.0.5" @@ -646,6 +741,7 @@ dependencies = [ "h3", "h3-quinn", "hmac", + "htmd", "http", "http-body", "http-body-util", @@ -654,6 +750,7 @@ dependencies = [ "hyper-util", "image", "ipnet", + "legible", "libc", "md-5", "memchr", @@ -742,6 +839,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -757,6 +860,16 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "futf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843" +dependencies = [ + "mac", + "new_debug_unreachable", +] + [[package]] name = "futures" version = "0.3.32" @@ -950,7 +1063,18 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "foldhash", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", ] [[package]] @@ -974,6 +1098,27 @@ dependencies = [ "digest", ] +[[package]] +name = "htmd" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e3283c6ef7b7af1e31cf13e9ce70ef2577c5a33a880bb80b3aaf791adfcc9a3" +dependencies = [ + "html5ever", + "markup5ever_rcdom", + "phf", +] + +[[package]] +name = "html5ever" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6452c4751a24e1b99c3260d505eaeee76a050573e61f30ac2c924ddc7236f01e" +dependencies = [ + "log", + "markup5ever", +] + [[package]] name = "http" version = "1.4.2" @@ -1332,6 +1477,22 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" +[[package]] +name = "legible" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63a92189b284405721b29a7a50f5024a8f04dc6eebba7ff92c2e79fbe91226b" +dependencies = [ + "dom_query", + "hashbrown 0.16.1", + "once_cell", + "regex", + "serde", + "serde_json", + "thiserror", + "url", +] + [[package]] name = "libc" version = "0.2.186" @@ -1365,6 +1526,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.30" @@ -1377,6 +1547,35 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "mac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" + +[[package]] +name = "markup5ever" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c3294c4d74d0742910f8c7b466f44dda9eb2d5742c1e430138df290a1e8451c" +dependencies = [ + "log", + "tendril", + "web_atoms", +] + +[[package]] +name = "markup5ever_rcdom" +version = "0.36.0+unofficial" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e5fc8802e8797c0dfdd2ce5c21aa0aee21abbc7b3b18559100651b3352a7b63" +dependencies = [ + "html5ever", + "markup5ever", + "tendril", + "xml5ever", +] + [[package]] name = "md-5" version = "0.11.0" @@ -1445,6 +1644,12 @@ dependencies = [ "pxfm", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + [[package]] name = "nom" version = "7.1.3" @@ -1531,6 +1736,29 @@ dependencies = [ "num-traits", ] +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + [[package]] name = "pem" version = "3.0.6" @@ -1547,6 +1775,59 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_macros", + "phf_shared", + "serde", +] + +[[package]] +name = "phf_codegen" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand", + "phf_shared", +] + +[[package]] +name = "phf_macros" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -1596,6 +1877,12 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + [[package]] name = "predicates" version = "3.1.4" @@ -1875,6 +2162,15 @@ dependencies = [ "yasna", ] +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + [[package]] name = "regex" version = "1.12.3" @@ -2054,6 +2350,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "security-framework" version = "3.7.0" @@ -2077,6 +2379,25 @@ dependencies = [ "libc", ] +[[package]] +name = "selectors" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fdfed56cd634f04fe8b9ddf947ae3dc493483e819593d2ba17df9ad05db8b2" +dependencies = [ + "bitflags", + "cssparser", + "derive_more", + "log", + "new_debug_unreachable", + "phf", + "phf_codegen", + "precomputed-hash", + "rustc-hash", + "servo_arc", + "smallvec", +] + [[package]] name = "semver" version = "1.0.28" @@ -2137,6 +2458,15 @@ dependencies = [ "zmij", ] +[[package]] +name = "servo_arc" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "170fb83ab34de17dc69aa7c67482b22218ddb85da56546f9bd6b929e32a05930" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "sha1" version = "0.11.0" @@ -2197,6 +2527,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + [[package]] name = "slab" version = "0.4.12" @@ -2225,6 +2561,30 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "string_cache" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a18596f8c785a729f2819c0f6a7eae6ebeebdfffbfe4214ae6b087f690e31901" +dependencies = [ + "new_debug_unreachable", + "parking_lot", + "phf_shared", + "precomputed-hash", +] + +[[package]] +name = "string_cache_codegen" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "585635e46db231059f76c5849798146164652513eb9e8ab2685939dd90f29b69" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", +] + [[package]] name = "strsim" version = "0.11.1" @@ -2304,6 +2664,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "tendril" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0" +dependencies = [ + "futf", + "mac", + "utf-8", +] + [[package]] name = "termtree" version = "0.5.1" @@ -2562,6 +2933,12 @@ dependencies = [ "serde", ] +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -2721,6 +3098,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web_atoms" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "075474b12bcb3d2e3d4546580e9de478eeeead668a1761e2a8860c836b7ef297" +dependencies = [ + "phf", + "phf_codegen", + "string_cache", + "string_cache_codegen", +] + [[package]] name = "webpki-root-certs" version = "1.0.7" @@ -3083,13 +3472,23 @@ dependencies = [ "rustix", ] +[[package]] +name = "xml5ever" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f57dd51b88a4b9f99f9b55b136abb86210629d61c48117ddb87f567e51e66be7" +dependencies = [ + "log", + "markup5ever", +] + [[package]] name = "yasna" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" dependencies = [ - "bit-vec", + "bit-vec 0.9.1", "time", ] diff --git a/Cargo.toml b/Cargo.toml index 09ea3442..1507b4fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ h2 = "=0.4.15" h3 = "=0.0.8" h3-quinn = "=0.0.10" hmac = "=0.13.0" +htmd = "=0.5.1" http = "=1.4.2" http-body = "=1.0.1" http-body-util = "=0.1.3" @@ -39,6 +40,7 @@ hyper = { version = "=1.10.1", features = ["client", "http1", "http2"] } hyper-util = { version = "=0.1.20", features = ["client-legacy", "client-proxy", "client-proxy-system", "http1", "http2", "tokio"] } image = { version = "=0.25.10", default-features = false, features = ["jpeg", "png", "tiff", "webp"] } ipnet = "=2.12.0" +legible = "=0.4.2" md-5 = "=0.11.0" memchr = "=2.8.3" mime = "=0.3.17" diff --git a/README.md b/README.md index 26bac714..186a9917 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ inspection, and request timing in one CLI. ## Features - **Response formatting** - Automatic formatting and syntax highlighting for JSON, XML, YAML, HTML, CSS, CSV, Markdown, MessagePack, Protocol Buffers, and more +- **HTML article to Markdown** - Extract readable content from HTML, convert it to Markdown, and add YAML frontmatter - **Image rendering** - Display images directly in your terminal - **WebSocket support** - Bidirectional WebSocket connections with automatic JSON formatting - **gRPC support** - Make gRPC calls with automatic reflection, discovery, and JSON-to-protobuf conversion @@ -109,6 +110,9 @@ fetch --session api https://example.com/dashboard # Convert a curl command into a fetch request fetch --from-curl 'curl -H "Authorization: Bearer TOKEN" https://api.example.com' +# Extract readable HTML and convert it to Markdown with YAML frontmatter +fetch --article https://example.com/post + # Show response headers, request+response headers, or full connection details fetch -v https://example.com fetch -vv https://example.com diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 2dae42c6..189d01aa 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -247,6 +247,33 @@ fetch -m HEAD example.com # Avoid body transfer when supported ## Formatting Options +### `--article` + +Convert an HTML response to Markdown by extracting its main readable content +with Legible and converting the extracted HTML with htmd. +If the response is already `text/markdown` or `text/x-markdown`, its body is +used directly. HTML output begins with YAML frontmatter containing the title, +byline, site name, published time, language, direction, text length, excerpt, +and final response URL when those details are available. Existing Markdown +receives frontmatter containing only the final response URL. + +```sh +fetch --article example.com/post +fetch --article --format off example.com/post > post.md +fetch --article -o post.md example.com/post +``` + +The HTML-to-Markdown conversion uses the final URL after redirects to resolve +relative HTML links. It requires buffering the decoded response and is limited +to 16 MiB. The flag cannot be combined with gRPC, WebSocket, `--discard`, +`--remote-name`, or +`--remote-header-name`. An explicit `Accept` request header overrides article +mode's HTML-oriented default. + +`--format`, `--color`, and `--pager` control terminal presentation of the +generated Markdown; they do not disable article extraction. Output files and +non-formatted pipes receive raw, uncolored Markdown. + ### `--format OPTION` Control response formatting. Values: `auto`, `on`, `off`. diff --git a/docs/output-formatting.md b/docs/output-formatting.md index a2e86340..ac77dca7 100644 --- a/docs/output-formatting.md +++ b/docs/output-formatting.md @@ -123,6 +123,31 @@ Features: fetch example.com ``` +#### Convert readable HTML to Markdown + +Use `--article` to extract the main readable content from an HTML response with +Legible and convert that extracted HTML to Markdown with htmd: + +```sh +fetch --article example.com/post +fetch --article --format off example.com/post > post.md +fetch --article -o post.md example.com/post +``` + +The converted HTML document starts with YAML frontmatter containing +available article metadata such as `title`, `byline`, `site_name`, +`published_time`, `lang`, `dir`, `length`, and `excerpt`. It also includes +`url`, set to the final response URL after redirects. Relative links are +resolved against that URL. + +If the response is already `text/markdown` or `text/x-markdown`, its Markdown +body is used directly and only `url` is added as frontmatter. + +Article extraction is a body transformation rather than terminal presentation: +`--format`, `--color`, and `--pager` only control how the resulting Markdown is +displayed. Output files receive raw, uncolored Markdown. Extraction requires +buffering the decoded HTML and therefore has a 16 MiB response limit. + ### CSS **Content-Type**: `text/css` diff --git a/src/cli.rs b/src/cli.rs index 68dfa37b..a15cb30b 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -108,6 +108,13 @@ pub struct Cli { #[arg(value_name = "URL", help = "The URL to make a request to")] pub url: Option, + #[arg( + long, + conflicts_with_all = ["discard", "grpc", "grpc_describe", "grpc_list", "remote_name"], + help = "Output readable HTML or Markdown" + )] + pub article: bool, + #[arg(last = true, hide = true)] pub extra_args: Vec, diff --git a/src/cli/completion.rs b/src/cli/completion.rs index 23c20985..9a32a6e0 100644 --- a/src/cli/completion.rs +++ b/src/cli/completion.rs @@ -198,6 +198,12 @@ const WS_INTERACTIVE_VALUES: &[FlagValue] = &[ ]; const FLAGS: &[Flag] = &[ + flag( + None, + "article", + "", + "Output readable HTML or Markdown with YAML frontmatter", + ), flag( None, "aws-sigv4", diff --git a/src/flag_registry.rs b/src/flag_registry.rs index 89e75fd5..35e94080 100644 --- a/src/flag_registry.rs +++ b/src/flag_registry.rs @@ -201,6 +201,7 @@ pub(crate) static FLAGS: &[FlagDef] = &[ }) .with_from_curl(), // ── Response ──────────────────────────────────────────────────────── + FlagDef::new("--article", Some(FlagCategory::Response), |c| c.article).with_ws_always(), FlagDef::new("--compress", Some(FlagCategory::Response), |c| { c.compress.is_some() }), diff --git a/src/format/article.rs b/src/format/article.rs new file mode 100644 index 00000000..6e9ddea0 --- /dev/null +++ b/src/format/article.rs @@ -0,0 +1,129 @@ +use legible::{Article, Options}; + +const MAX_ARTICLE_ELEMENTS: usize = 500_000; + +pub fn extract_markdown(html: &str, url: &str) -> Result, String> { + let options = Options::new().max_elems_to_parse(MAX_ARTICLE_ELEMENTS); + let article = legible::parse(html, Some(url), Some(options)) + .map_err(|err| format!("failed to extract readable article: {err}"))?; + let markdown = htmd::convert(&article.content) + .map_err(|err| format!("failed to convert extracted article to Markdown: {err}"))?; + Ok(render_document(&article, url, &markdown).into_bytes()) +} + +pub fn add_url_frontmatter(markdown: &str, url: &str) -> Vec { + let mut out = String::from("---\n"); + push_string(&mut out, "url", url); + out.push_str("---\n\n"); + out.push_str(markdown); + if !markdown.ends_with('\n') { + out.push('\n'); + } + out.into_bytes() +} + +fn render_document(article: &Article, url: &str, markdown: &str) -> String { + let mut out = String::from("---\n"); + push_string(&mut out, "title", &article.title); + push_optional_string(&mut out, "byline", article.byline.as_deref()); + push_optional_string(&mut out, "site_name", article.site_name.as_deref()); + push_optional_string( + &mut out, + "published_time", + article.published_time.as_deref(), + ); + push_optional_string(&mut out, "lang", article.lang.as_deref()); + push_optional_string(&mut out, "dir", article.dir.as_deref()); + out.push_str(&format!("length: {}\n", article.length)); + push_optional_string(&mut out, "excerpt", article.excerpt.as_deref()); + push_string(&mut out, "url", url); + out.push_str("---\n\n"); + out.push_str(markdown.trim()); + out.push('\n'); + out +} + +fn push_optional_string(out: &mut String, key: &str, value: Option<&str>) { + if let Some(value) = value { + push_string(out, key, value); + } +} + +fn push_string(out: &mut String, key: &str, value: &str) { + let value = serde_json::to_string(value).expect("strings always serialize as JSON"); + out.push_str(key); + out.push_str(": "); + out.push_str(&value); + out.push('\n'); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn extracts_article_as_markdown_with_safe_frontmatter() { + let html = r#" + +An "Quoted" Article + + + + + +

An Article

This is a sufficiently substantial article paragraph with readable content for extraction. It contains enough prose to be selected as the primary document content.

Related reading

+"#; + + let output = extract_markdown(html, "https://example.com/posts/one").unwrap(); + let output = String::from_utf8(output).unwrap(); + + assert!(output.starts_with("---\n")); + assert!(output.contains("title: \"An ")); + assert!(output.contains("byline: \"Jane Smith\"")); + assert!(output.contains("site_name: \"Example News\"")); + assert!(output.contains("lang: \"en\"")); + assert!(output.contains("url: \"https://example.com/posts/one\"")); + assert!(output.contains("primary document content"), "{output}"); + assert!( + output.contains("https://example.com/posts/relative"), + "{output}" + ); + assert!(!output.contains("Navigation that should be removed")); + assert!(output.ends_with('\n')); + assert!(!output.ends_with("\n\n")); + } + + #[test] + fn markdown_frontmatter_preserves_the_document() { + let output = add_url_frontmatter( + "# Existing Markdown\n\nBody with *formatting*.\n", + "https://example.com/readme.md", + ); + assert_eq!( + String::from_utf8(output).unwrap(), + "---\nurl: \"https://example.com/readme.md\"\n---\n\n# Existing Markdown\n\nBody with *formatting*.\n" + ); + } + + #[test] + fn frontmatter_quotes_multiline_values() { + let article = Article { + title: "title: ---\nnext".to_string(), + byline: None, + dir: None, + lang: None, + content: String::new(), + text_content: String::new(), + length: 0, + excerpt: None, + site_name: None, + published_time: None, + }; + + let output = render_document(&article, "https://example.com/?a=1&b=2", "Body"); + assert!(output.contains(r#"title: "title: ---\nnext""#)); + assert!(!output.contains("byline:")); + assert_eq!(output.matches("---\n").count(), 2); + assert!(output.ends_with("Body\n")); + } +} diff --git a/src/format/mod.rs b/src/format/mod.rs index 203b4629..2a5f0017 100644 --- a/src/format/mod.rs +++ b/src/format/mod.rs @@ -1,3 +1,4 @@ +pub mod article; pub mod content_type; pub mod css; pub mod csv; diff --git a/src/http/mod.rs b/src/http/mod.rs index 9d1b3a38..4464f1f4 100644 --- a/src/http/mod.rs +++ b/src/http/mod.rs @@ -176,10 +176,12 @@ async fn execute_inner(cli: &Cli) -> Result { apply_headers(&mut headers, &cli.headers)?; grpc_headers::apply_standard_headers(&mut headers); } else { - headers.insert( - ACCEPT, - HeaderValue::from_static(core::DEFAULT_ACCEPT_HEADER), - ); + let accept = if cli.article { + "text/html, application/xhtml+xml;q=0.9, text/markdown;q=0.8, */*;q=0.1" + } else { + core::DEFAULT_ACCEPT_HEADER + }; + headers.insert(ACCEPT, HeaderValue::from_static(accept)); apply_headers(&mut headers, &cli.headers)?; } apply_ranges(&mut headers, &cli.ranges); diff --git a/src/http/response.rs b/src/http/response.rs index 768744d8..8d92d303 100644 --- a/src/http/response.rs +++ b/src/http/response.rs @@ -20,10 +20,10 @@ use metadata::{ body_duration, check_grpc_status, exit_code, finalize_streamed_response, handle_clipboard_outcome, print_response_metadata, print_timing, }; -use stdout::{stdout_stream_target, write_stdout_bytes}; +use stdout::{StdoutBody, stdout_stream_target, write_stdout_bytes}; use stream::{ - read_decoded_response_body_limited, stream_response_to_discard, stream_response_to_output, - stream_response_to_stdout, + read_decoded_article_body_limited, read_decoded_response_body_limited, + stream_response_to_discard, stream_response_to_output, stream_response_to_stdout, }; pub(super) async fn finish_response( @@ -72,6 +72,20 @@ pub(super) async fn finish_response( if let Some(warning) = &resolved_output.warning { write_warning(cli, warning); } + if cli.article { + return finish_article_response( + cli, + response, + response_headers, + response_url, + compression, + status, + response_timing, + method_is_head, + resolved_output.path.as_deref(), + ) + .await; + } if let Some(path) = resolved_output.path { let progress = if cli.silent { output::WriteProgress::disabled() @@ -203,3 +217,117 @@ pub(super) async fn finish_response( let code = exit_code(status.as_u16(), cli.ignore_status); Ok(check_grpc_status(cli, &response_headers, &trailers, code)) } + +#[allow(clippy::too_many_arguments)] +async fn finish_article_response( + cli: &Cli, + response: Response, + response_headers: HeaderMap, + response_url: url::Url, + compression: CompressionMode, + status: StatusCode, + response_timing: Option, + method_is_head: bool, + output_path: Option<&str>, +) -> Result { + let body_start = Instant::now(); + let (bytes, trailers) = + read_decoded_article_body_limited(response, response_headers.clone(), compression).await?; + let body_duration = body_duration(method_is_head, &bytes, body_start); + + let input_kind = article_response_content_kind(&response_headers, &bytes); + if input_kind == ArticleInputKind::Unsupported { + let content_type = stdout::response_header_content_type_label(&response_headers); + return Err(FetchError::Message(format!( + "response content type '{content_type}' is not supported with '--article'; try again without '--article'" + ))); + } + + let raw_content_type = response_headers + .get(CONTENT_TYPE) + .and_then(|value| value.to_str().ok()); + let (_, charset) = content_type::get_content_type(raw_content_type); + let text = formatters::transcode_bytes(&bytes, &charset); + let text = String::from_utf8_lossy(&text); + let article = match input_kind { + ArticleInputKind::Html => { + crate::format::article::extract_markdown(&text, response_url.as_str()) + .map_err(FetchError::Message)? + } + ArticleInputKind::Markdown => { + crate::format::article::add_url_frontmatter(&text, response_url.as_str()) + } + ArticleInputKind::Unsupported => unreachable!("unsupported article input returned above"), + }; + + if cli.copy { + handle_clipboard_outcome(cli, clipboard::copy_bytes(&article)); + } + + if let Some(path) = output_path { + let progress = if cli.silent { + output::WriteProgress::disabled() + } else { + output::WriteProgress::stdio( + cli.color.as_deref(), + Some(i64::try_from(article.len()).unwrap_or(i64::MAX)), + ) + }; + output::write_output_with_progress(path, &article, cli.clobber, progress) + .await + .map_err(|err| FetchError::Message(err.to_string()))?; + } else { + let stdout_is_terminal = core::stdio().stdout_is_terminal(); + let rendered = if core::format_enabled(cli.format.as_deref(), stdout_is_terminal) { + let use_color = core::color_enabled(cli.color.as_deref(), stdout_is_terminal); + let mut out = core::Printer::new(use_color); + if markdown::format_markdown_to(&article, &mut out).is_ok() { + out.into_bytes() + } else { + article.clone() + } + } else { + article.clone() + }; + write_stdout_bytes( + cli, + &StdoutBody { + bytes: rendered, + content_type: ContentType::Markdown, + content_type_label: "text/markdown; charset=utf-8".to_string(), + }, + )?; + } + + print_timing(cli, response_timing, body_duration); + let code = exit_code(status.as_u16(), cli.ignore_status); + Ok(check_grpc_status(cli, &response_headers, &trailers, code)) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ArticleInputKind { + Html, + Markdown, + Unsupported, +} + +fn article_response_content_kind(headers: &HeaderMap, bytes: &[u8]) -> ArticleInputKind { + let content_type = headers + .get(CONTENT_TYPE) + .and_then(|value| value.to_str().ok()); + if content_type::get_content_type(content_type).0 == ContentType::Markdown { + return ArticleInputKind::Markdown; + } + + let declared_html = content_type + .and_then(|value| value.parse::().ok()) + .is_some_and(|mime| { + (mime.type_() == mime::TEXT && mime.subtype() == mime::HTML) + || (mime.type_() == mime::APPLICATION && mime.subtype().as_str() == "xhtml") + }); + if declared_html || content_type::sniff_content_type(bytes) == ContentType::Html { + ArticleInputKind::Html + } else { + ArticleInputKind::Unsupported + } +} diff --git a/src/http/response/stream.rs b/src/http/response/stream.rs index 1e97a9a0..1907e74c 100644 --- a/src/http/response/stream.rs +++ b/src/http/response/stream.rs @@ -15,6 +15,35 @@ pub(super) async fn read_decoded_response_body_limited( response: Response, response_headers: HeaderMap, compression: CompressionMode, +) -> Result<(Vec, HeaderMap), FetchError> { + read_decoded_response_body_with_limit_message( + response, + response_headers, + compression, + "cannot be buffered; use '--format off' or write to a file to stream it", + ) + .await +} + +pub(super) async fn read_decoded_article_body_limited( + response: Response, + response_headers: HeaderMap, + compression: CompressionMode, +) -> Result<(Vec, HeaderMap), FetchError> { + read_decoded_response_body_with_limit_message( + response, + response_headers, + compression, + "cannot be extracted as an article", + ) + .await +} + +async fn read_decoded_response_body_with_limit_message( + response: Response, + response_headers: HeaderMap, + compression: CompressionMode, + limit_message: &str, ) -> Result<(Vec, HeaderMap), FetchError> { let (reader, trailers) = async_response_reader(response); let mut reader = decoded_async_response_reader(reader, compression, &response_headers)?; @@ -28,7 +57,7 @@ pub(super) async fn read_decoded_response_body_limited( } if bytes.len().saturating_add(n) > MAX_BUFFERED_RESPONSE_BYTES { return Err(FetchError::Message(format!( - "response body exceeds {} bytes and cannot be buffered; use '--format off' or write to a file to stream it", + "response body exceeds {} bytes and {limit_message}", MAX_BUFFERED_RESPONSE_BYTES ))); } diff --git a/tests/formatting.rs b/tests/formatting.rs index ec716bb2..d0106c8f 100644 --- a/tests/formatting.rs +++ b/tests/formatting.rs @@ -66,6 +66,107 @@ fn response_formatting_json_ndjson_xml_yaml_html_csv_css_and_sniffing() { } } +#[test] +fn article_mode_extracts_markdown_frontmatter_and_uses_html_accept() { + let server = TestServer::start(|_| { + TestResponse::ok( + r#" +Readable Article + + + +
+

Readable Article

+

This is the primary article content, with enough prose to be recognized as the readable part of the page rather than surrounding navigation or other incidental elements.

+

Continue with related reading for more detail about this topic and its wider context.

+
"#, + ) + .header("Content-Type", "text/html; charset=utf-8") + }); + + let res = run_fetch(&[&format!("{}/posts/one", server.url), "--article"]); + + assert_exit(&res, 0); + assert!(res.stdout.starts_with("---\n"), "{}", res.stdout); + assert!(res.stdout.contains("title: \"Readable Article\"")); + assert!(res.stdout.contains("byline: \"Jane Smith\"")); + assert!(res.stdout.contains("site_name: \"Example News\"")); + assert!( + res.stdout + .contains(&format!("url: \"{}/posts/one\"", server.url)) + ); + assert!(res.stdout.contains("primary article content")); + assert!( + res.stdout + .contains(&format!("[related reading]({}/posts/related)", server.url)), + "{}", + res.stdout + ); + assert!(!res.stdout.contains("Site navigation")); + assert!(!res.stdout.contains("Unrelated sidebar")); + assert!(res.stdout.ends_with('\n')); + + let requests = server.requests(); + assert_eq!( + requests[0].header("accept"), + "text/html, application/xhtml+xml;q=0.9, text/markdown;q=0.8, */*;q=0.1" + ); +} + +#[test] +fn article_mode_writes_transformed_output_file_and_rejects_non_html() { + let server = TestServer::start(|req| { + match req.path.as_str() { + "/article" => TestResponse::ok( + "Saved Article

Saved Article

A long and useful paragraph containing the readable document body that should be converted and saved as Markdown output.

", + ) + .header("Content-Type", "text/html"), + "/markdown" => TestResponse::ok( + "# Existing Markdown\n\nThis document should pass through *unchanged*.\n", + ) + .header("Content-Type", "text/markdown; charset=utf-8"), + _ => TestResponse::ok(r#"{"ok":true}"#).header("Content-Type", "application/json"), + } + }); + let dir = TempDir::new().unwrap(); + let path = dir.path().join("article.md"); + + let res = run_fetch(&[ + &format!("{}/article", server.url), + "--article", + "--output", + path.to_str().unwrap(), + ]); + assert_exit(&res, 0); + assert!(res.stdout.is_empty()); + let saved = fs::read_to_string(path).unwrap(); + assert!(saved.contains("title: \"Saved Article\"")); + assert!(saved.contains("readable document body")); + assert!(!saved.contains("
")); + + let markdown_url = format!("{}/markdown", server.url); + let res = run_fetch(&[&markdown_url, "--article"]); + assert_exit(&res, 0); + assert_eq!( + res.stdout, + format!( + "---\nurl: {markdown_url:?}\n---\n\n# Existing Markdown\n\nThis document should pass through *unchanged*.\n" + ) + ); + assert!(!res.stdout.contains("title:")); + assert!(!res.stdout.contains("length:")); + + let res = run_fetch(&[&format!("{}/json", server.url), "--article"]); + assert_exit(&res, 1); + assert!( + res.stderr.contains( + "response content type 'application/json' is not supported with '--article'; try again without '--article'" + ), + "{}", + res.stderr + ); +} + #[cfg(unix)] #[test] fn formatted_sse_outputs_events_before_stream_ends() {