From 2091d8d703dc61039c207718090822357100d731 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Mon, 31 Aug 2026 16:36:47 +0800 Subject: [PATCH 1/4] feat(pytorch): Add PyTorch 2.10 through 2.13 support Add the four newer documentation releases and extract complete structured breadcrumbs used by the current PyTorch documentation theme. Keep the existing HTTP-based scraping workflow and cover structured, malformed, single-item, and legacy breadcrumb formats. --- lib/docs/filters/pytorch/entries.rb | 31 +++++++++++++++++++++++++---- lib/docs/scrapers/pytorch.rb | 22 +++++++++++++++++++- 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/lib/docs/filters/pytorch/entries.rb b/lib/docs/filters/pytorch/entries.rb index 85d2f06418..7b31d037d4 100644 --- a/lib/docs/filters/pytorch/entries.rb +++ b/lib/docs/filters/pytorch/entries.rb @@ -2,6 +2,9 @@ module Docs class Pytorch class EntriesFilter < Docs::EntriesFilter def get_breadcrumbs + breadcrumbs = schema_breadcrumbs + return breadcrumbs unless breadcrumbs.empty? + breadcrumbs = if at_css('.pytorch-breadcrumbs') css('.pytorch-breadcrumbs > li').map { |node| node.content.delete_suffix(' >').strip @@ -26,15 +29,19 @@ def get_name end def get_type - if at_css('.pytorch-breadcrumbs') - get_breadcrumbs[1] + breadcrumbs = get_breadcrumbs + + if schema_breadcrumbs.any? + breadcrumbs.size > 2 ? breadcrumbs[-2] : breadcrumbs[-1] + elsif at_css('.pytorch-breadcrumbs') + breadcrumbs[1] else - get_breadcrumbs.size > 2 ? get_breadcrumbs[2] : get_breadcrumbs[1] + breadcrumbs.size > 2 ? breadcrumbs[2] : breadcrumbs[1] end end def include_default_entry? - !get_breadcrumbs.nil? && get_breadcrumbs.size >= 2 + schema_breadcrumbs.any? || get_breadcrumbs.size >= 2 end def additional_entries @@ -61,6 +68,22 @@ def additional_entries entries end + + private + + def schema_breadcrumbs + @schema_breadcrumbs ||= css( + '[itemtype="https://schema.org/BreadcrumbList"] [itemprop="itemListElement"]' + ).filter_map do |node| + name = node.at_css('[itemprop="name"]') + next unless name + + text = Nokogiri::HTML.fragment(name['content'] || name.content).text.strip + dangling_text = node.text.strip.delete_suffix('">') if name['content'] + text = "#{text} #{dangling_text}" if dangling_text.present? + text + end.reject(&:empty?) + end end end end diff --git a/lib/docs/scrapers/pytorch.rb b/lib/docs/scrapers/pytorch.rb index e45ac3b096..f1d797ab93 100644 --- a/lib/docs/scrapers/pytorch.rb +++ b/lib/docs/scrapers/pytorch.rb @@ -15,10 +15,30 @@ class Pytorch < UrlScraper options[:max_image_size] = 1_000_000 options[:attribution] = <<-HTML - © 2025, PyTorch Contributors
+ © 2026, PyTorch Contributors
PyTorch has a BSD-style license, as found in the LICENSE file. HTML + version '2.13' do + self.release = '2.13' + self.base_url = "https://docs.pytorch.org/docs/#{release}/" + end + + version '2.12' do + self.release = '2.12' + self.base_url = "https://docs.pytorch.org/docs/#{release}/" + end + + version '2.11' do + self.release = '2.11' + self.base_url = "https://docs.pytorch.org/docs/#{release}/" + end + + version '2.10' do + self.release = '2.10' + self.base_url = "https://docs.pytorch.org/docs/#{release}/" + end + version '2.9' do self.release = '2.9' self.base_url = "https://docs.pytorch.org/docs/#{release}/" From cfe93fbc391ac50a5e8033479b9a12a5c6ef59e3 Mon Sep 17 00:00:00 2001 From: Simon Legner Date: Sun, 13 Sep 2026 12:33:44 +0200 Subject: [PATCH 2/4] Update PyTorch documentation (2.14) --- lib/docs/scrapers/pytorch.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/docs/scrapers/pytorch.rb b/lib/docs/scrapers/pytorch.rb index f1d797ab93..edc60704b4 100644 --- a/lib/docs/scrapers/pytorch.rb +++ b/lib/docs/scrapers/pytorch.rb @@ -19,6 +19,11 @@ class Pytorch < UrlScraper PyTorch has a BSD-style license, as found in the LICENSE file. HTML + version '2.14' do + self.release = '2.14' + self.base_url = "https://docs.pytorch.org/docs/#{release}/" + end + version '2.13' do self.release = '2.13' self.base_url = "https://docs.pytorch.org/docs/#{release}/" From 2435bddb131de38cb07bb149e9c4acf0894ce364 Mon Sep 17 00:00:00 2001 From: Simon Legner Date: Sun, 13 Sep 2026 12:41:23 +0200 Subject: [PATCH 3/4] feat(pytorch): Convert to FileScraper Scrape the PyTorch documentation from a local copy instead of fetching thousands of pages from docs.pytorch.org on every run. The rendered HTML is published per version in the "site" branch of pytorch/docs, so document the sparse checkout in docs/file-scrapers.md. Skip non-HTML paths, since the published tree ships a .md source next to every .html page. --- docs/file-scrapers.md | 13 +++++++++++++ lib/docs/scrapers/pytorch.rb | 14 ++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/docs/file-scrapers.md b/docs/file-scrapers.md index 3d2dea25fb..051ed84c98 100644 --- a/docs/file-scrapers.md +++ b/docs/file-scrapers.md @@ -227,6 +227,19 @@ curl -L https://docs.python.org/ftp/python/doc/$RELEASE/python-$RELEASE-docs-htm tar xj --strip-components=1 ``` +## PyTorch + +The rendered documentation is published in the `site` branch of +https://github.com/pytorch/docs, with one directory per version. + +```sh +git clone --branch site --depth 1 --filter=blob:none --sparse \ +https://github.com/pytorch/docs.git /tmp/pytorch-docs; \ +git -C /tmp/pytorch-docs sparse-checkout set $VERSION; \ +mv /tmp/pytorch-docs/$VERSION docs/pytorch~$VERSION; \ +rm -rf /tmp/pytorch-docs +``` + ## R ```bash diff --git a/lib/docs/scrapers/pytorch.rb b/lib/docs/scrapers/pytorch.rb index edc60704b4..836b8589a2 100644 --- a/lib/docs/scrapers/pytorch.rb +++ b/lib/docs/scrapers/pytorch.rb @@ -1,8 +1,12 @@ module Docs - class Pytorch < UrlScraper + # Requires downloading the documents to local disk first. + # The rendered HTML lives in the "site" branch of https://github.com/pytorch/docs, + # one directory per version; see docs/file-scrapers.md for the commands. + class Pytorch < FileScraper self.name = 'PyTorch' self.slug = 'pytorch' self.type = 'sphinx' + self.root_path = 'index.html' self.links = { home: 'https://pytorch.org/', code: 'https://github.com/pytorch/pytorch' @@ -11,7 +15,13 @@ class Pytorch < UrlScraper html_filters.push 'pytorch/entries', 'pytorch/clean_html', 'sphinx/clean_html' options[:skip] = ['cpp_index.html', 'deploy.html', 'packages.html', 'py-modindex.html', 'genindex.html'] - options[:skip_patterns] = [/\Acommunity/, /\A_modules/, /\Anotes/, /\Aorg\/pytorch\//] + options[:skip_patterns] = [ + /.*(? Date: Sun, 13 Sep 2026 12:43:23 +0200 Subject: [PATCH 4/4] feat(pytorch): Download the documentation files Check out the version being scraped from the "site" branch of pytorch/docs, which is what https://docs.pytorch.org serves. There is no documentation archive to download, and the branch holds every version at once, so the files are taken from a sparse clone rather than with #download_and_extract. --- docs/file-scrapers.md | 13 ------------- lib/docs/scrapers/pytorch.rb | 26 +++++++++++++++++++++++--- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/docs/file-scrapers.md b/docs/file-scrapers.md index 42355874ce..a6cbce6174 100644 --- a/docs/file-scrapers.md +++ b/docs/file-scrapers.md @@ -85,19 +85,6 @@ dpkg -x $PACKAGE ./ mv ./usr/share/doc/openjdk-$VERSION-jre-headless/api/ docs/openjdk~$VERSION ``` -## PyTorch - -The rendered documentation is published in the `site` branch of -https://github.com/pytorch/docs, with one directory per version. - -```sh -git clone --branch site --depth 1 --filter=blob:none --sparse \ -https://github.com/pytorch/docs.git /tmp/pytorch-docs; \ -git -C /tmp/pytorch-docs sparse-checkout set $VERSION; \ -mv /tmp/pytorch-docs/$VERSION docs/pytorch~$VERSION; \ -rm -rf /tmp/pytorch-docs -``` - ## R ```bash diff --git a/lib/docs/scrapers/pytorch.rb b/lib/docs/scrapers/pytorch.rb index 836b8589a2..3e9c3b67be 100644 --- a/lib/docs/scrapers/pytorch.rb +++ b/lib/docs/scrapers/pytorch.rb @@ -1,7 +1,4 @@ module Docs - # Requires downloading the documents to local disk first. - # The rendered HTML lives in the "site" branch of https://github.com/pytorch/docs, - # one directory per version; see docs/file-scrapers.md for the commands. class Pytorch < FileScraper self.name = 'PyTorch' self.slug = 'pytorch' @@ -112,5 +109,28 @@ class Pytorch < FileScraper def get_latest_version(opts) get_latest_github_release('pytorch', 'pytorch', opts) end + + private + + # There is no documentation archive, the files have to be taken from the + # repository hosting https://docs.pytorch.org. + def download_source + require 'tmpdir' + + Dir.mktmpdir do |directory| + repository = File.join(directory, 'docs') + + instrument 'info.doc', msg: %(Cloning the PyTorch #{self.class.version} documentation...) + # The "site" branch holds the rendered documentation of every version, + # of which only the one being scraped is checked out. + system('git', 'clone', '--branch', 'site', '--depth', '1', '--filter=blob:none', '--sparse', + 'https://github.com/pytorch/docs', repository) + system('git', '-C', repository, 'sparse-checkout', 'set', self.class.version) + + instrument 'info.doc', msg: %(Moving the documentation files to "#{source_directory}"...) + FileUtils.mkpath(File.dirname(source_directory)) + FileUtils.mv(File.join(repository, self.class.version), source_directory) + end + end end end