diff --git a/Gemfile b/Gemfile index 341849522e..8940f4f521 100644 --- a/Gemfile +++ b/Gemfile @@ -35,6 +35,8 @@ group :development do end group :docs do + gem 'kramdown' + gem 'kramdown-parser-gfm' gem 'redcarpet' end diff --git a/Gemfile.lock b/Gemfile.lock index a9e51e34b3..20eea3352a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -134,6 +134,10 @@ GEM rdoc (>= 4.0.0) reline (>= 0.4.2) json (2.19.4) + kramdown (2.5.2) + rexml (>= 3.4.4) + kramdown-parser-gfm (1.1.0) + kramdown (~> 2.0) logger (1.7.0) loofah (2.25.1) crass (~> 1.0.2) @@ -311,6 +315,8 @@ DEPENDENCIES html-pipeline (~> 2.14) image_optim image_optim_pack + kramdown + kramdown-parser-gfm minitest newrelic_rpm nokogiri @@ -390,6 +396,8 @@ CHECKSUMS io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc irb (1.18.0) sha256=de9454a0703a54704b9811a5ef31a60c86949fbf4013fcf244fabc7c775248e3 json (2.19.4) sha256=670a7d333fb3b18ca5b29cb255eb7bef099e40d88c02c80bd42a3f30fe5239ac + kramdown (2.5.2) sha256=1ba542204c66b6f9111ff00dcc26075b95b220b07f2905d8261740c82f7f02fa + kramdown-parser-gfm (1.1.0) sha256=fb39745516427d2988543bf01fc4cf0ab1149476382393e0e9c48592f6581729 logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203 loofah (2.25.1) sha256=d436c73dbd0c1147b16c4a41db097942d217303e1f7728704b37e4df9f6d2e04 method_source (1.1.0) sha256=181301c9c45b731b4769bc81e8860e72f9161ad7d66dd99103c9ab84f560f5c5 diff --git a/lib/docs/core/scrapers/file_scraper.rb b/lib/docs/core/scrapers/file_scraper.rb index 07986b5106..205f87c542 100644 --- a/lib/docs/core/scrapers/file_scraper.rb +++ b/lib/docs/core/scrapers/file_scraper.rb @@ -199,13 +199,16 @@ def download_source end # Downloads an archive and moves it into #source_directory. Pass the - # subdirectory holding the documents when they aren't at the archive's root. - def download_and_extract(url, subdirectory = nil) + # subdirectory holding the documents when they aren't at the archive's root, + # and a destination to unpack somewhere else than #source_directory. Note + # that the destination is replaced, so the one holding the documents has to + # be unpacked before those nested inside it. + def download_and_extract(url, subdirectory = nil, destination: source_directory) instrument 'info.doc', msg: %(Downloading #{url}...) archive = Archive.download(url) - instrument 'info.doc', msg: %(Extracting the documentation files to "#{source_directory}"...) - Archive.unpack(archive, source_directory, directory: subdirectory) + instrument 'info.doc', msg: %(Extracting the documentation files to "#{destination}"...) + Archive.unpack(archive, destination, directory: subdirectory) ensure FileUtils.rm_f(archive) if archive end diff --git a/lib/docs/filters/css/clean_html.rb b/lib/docs/filters/css/clean_html.rb deleted file mode 100644 index f202f7c96d..0000000000 --- a/lib/docs/filters/css/clean_html.rb +++ /dev/null @@ -1,32 +0,0 @@ -module Docs - class Css - class CleanHtmlFilter < Filter - def call - root_page? ? root : other - doc - end - - def root - # Remove "CSS3 Tutorials" and everything after - css('#CSS3_Tutorials ~ *', '#CSS3_Tutorials').remove - end - - def other - css('.syntaxbox > .syntaxbox').each do |node| - node.parent.before(node.parent.children).remove - end - - # Remove "|" and "||" links in syntax box (e.g. animation, all, etc.) - css('.syntaxbox', '.twopartsyntaxbox').css('a').each do |node| - if node.content == '|' || node.content == '||' - node.replace node.content - end - end - - css('img[style*="float"]').each do |node| - node['style'] = node['style'] + ';float: none; display: block;' - end - end - end - end -end diff --git a/lib/docs/filters/css/entries.rb b/lib/docs/filters/css/entries.rb index 0727a6b826..cb71d1e73c 100644 --- a/lib/docs/filters/css/entries.rb +++ b/lib/docs/filters/css/entries.rb @@ -1,146 +1,78 @@ module Docs class Css class EntriesFilter < Docs::EntriesFilter - TYPE_BY_PATH = { - 'CSS_Animations' => 'Animations & Transitions', - 'CSS_Background_and_Borders' => 'Backgrounds & Borders', - 'CSS_Columns' => 'Multi-column Layout', - 'CSS_Flexible_Box_Layout' => 'Flexible Box Layout', - 'CSS_Fonts' => 'Fonts', - 'CSS_Grid_Layout' => 'Grid Layout', - 'CSS_Images' => 'Images', - 'CSS_Lists_and_Counters' => 'Lists', - 'CSS_Transforms' => 'Transforms', - 'Media_Queries' => 'Media Queries', - 'filter-function' => 'Filter Effects', - 'transform-function' => 'Transforms', - '@media' => 'Media Queries', - 'overscroll' => 'Overscroll', - 'text-size-adjust' => 'Miscellaneous', - 'resolved_value' => 'Miscellaneous', - 'touch-action' => 'Miscellaneous', - 'will-change' => 'Miscellaneous' + # What the specifications call themselves and what the documentation + # files them under. + TYPE_BY_SPEC = { + 'Compositing' => 'Compositing & Blending', + 'Custom Properties for Cascading Variables' => 'Variables', + 'Grid Layout' => 'Grid Layout', + 'Image Values & Replaced Content' => 'Image Values', + 'Scroll Snap' => 'Scroll Snap' } - DATA_TYPE_SLUGS = %w(angle basic-shape color_value counter frequency - gradient image integer length number percentage position_value ratio - resolution shape string time timing-function uri user-ident) + # The kind of page to fall back on when no specification names it. + TYPE_BY_PAGE_TYPE = { + 'css-at-rule' => 'At-rules', + 'css-at-rule-descriptor' => 'At-rules', + 'css-combinator' => 'Selectors', + 'css-function' => 'Functions', + 'css-keyword' => 'Keywords', + 'css-media-feature' => 'Media Queries', + 'css-property' => 'Properties', + 'css-pseudo-class' => 'Selectors', + 'css-pseudo-element' => 'Pseudo-Elements', + 'css-selector' => 'Selectors', + 'css-shorthand-property' => 'Properties', + 'css-type' => 'Data Types' + } + + VENDOR_PREFIXES = %w(-webkit- -moz- -ms- -o-) - FUNCTION_SLUGS = %w(attr calc cross-fade cubic-bezier cycle element - linear-gradient radial-gradient repeating-linear-gradient - repeating-radial-gradient var) + # The descriptors and the media features of an at-rule go by their own + # name in the content repository; the reference lists them under the + # at-rule they belong to. + NESTED = %w(css-at-rule-descriptor css-media-feature) def get_name - if DATA_TYPE_SLUGS.include?(slug) - "<#{super.remove ' value'}>" - elsif FUNCTION_SLUGS.include?(slug) - "#{super}()" - elsif slug =~ /\A[a-z]+_/i - slug.to_s.gsub('_', ' ').gsub('/', ': ') - elsif slug.start_with?('transform-function') || slug.start_with?('filter-function') - slug.split('/').last + '()' - else - super - end + name = page.front_matter['short-title'] || default_name + at_rule ? "#{at_rule}.#{name}" : name end - def get_type - if slug.include?('-webkit') || slug.include?('-moz') || slug.include?('-ms') - 'Extensions' - elsif type = TYPE_BY_PATH[slug.split('/').first] - type - elsif type = get_spec - type.remove! 'CSS ' - type.remove! ' Module' - type.remove! %r{ Level \d\z} - type.remove! %r{\(.*\)} - type.remove! %r{ \d\z} - type.sub! 'and', '&' - type.strip! - type = 'Grid Layout' if type.include?('Grid Layout') - type = 'Scroll Snap' if type.include?('Scroll Snap') - type = 'Compositing & Blending' if type.include?('Compositing') - type = 'Animations & Transitions' if type.in?(%w(Animations Transitions)) - type = 'Image Values' if type == 'Image Values & Replaced Content' - type = 'Variables' if type == 'Custom Properties for Cascading Variables' - type.prepend 'Miscellaneous ' if type =~ /\ALevel \d\z/ - type - elsif name.start_with?('::') - 'Pseudo-Elements' - elsif name.start_with?(':') - 'Selectors' - elsif name.start_with?('display-') - 'Display' - else - 'Miscellaneous' - end + def default_name + page.page_type.to_s.start_with?('css-') ? slug.split('/').last : page.title end - STATUSES = { - 'spec-Living' => 0, - 'spec-REC' => 1, - 'spec-CR' => 2, - 'spec-PR' => 3, - 'spec-LC' => 4, - 'spec-WD' => 5, - 'spec-ED' => 6, - 'spec-Obsolete' => 7 - } - - PRIORITY_STATUSES = %w(spec-REC spec-CR) - PRIORITY_SPECS = ['CSS Basic Box Model', 'CSS Lists and Counters', 'CSS Paged Media'] - - def get_spec - return unless table = at_css('#Specifications + table') || css('.standard-table').last - - specs = table.css('tbody tr').to_a - # [link, span] - specs.map! { |node| [node.at_css('> td:nth-child(1) > a'), node.at_css('> td:nth-child(2) > span')] } - # ignore non-CSS specs - specs.select! { |pair| pair.first && pair.first['href'] =~ /css|fxtf|fullscreen|svg/i && !pair.first['href'].include?('compat.spec') } - # ignore specs with no status - specs.select! { |pair| pair.second } - # ["Spec", "spec-REC"] - specs.map! { |pair| [pair.first.child.content, pair.second['class']] } - # sort by status - specs.sort_by! { |pair| [STATUSES[pair.second], pair.first] } - - spec = specs.find { |pair| PRIORITY_SPECS.any? { |s| pair.first.start_with?(s) } && name != 'display' } - spec ||= specs.find { |pair| !pair.first.start_with?('CSS Level') && pair.second.in?(PRIORITY_STATUSES) } - spec ||= specs.find { |pair| pair.second == 'spec-WD' } if specs.count { |pair| pair.second == 'spec-WD' } == 1 - spec ||= specs.first + def at_rule + return unless NESTED.include?(page.page_type) + context[:pages][slug.split('/')[0..-2].join('/')]&.short_title + end - spec.try(:first) + def get_type + return 'Extensions' if VENDOR_PREFIXES.any? { |prefix| name.include?(prefix) } + specification || TYPE_BY_PAGE_TYPE[page.page_type] || 'Miscellaneous' end - ADDITIONAL_ENTRIES = { - 'shape' => [ - %w(rect() Syntax) ], - 'timing-function' => [ - %w(cubic-bezier()), - %w(steps()), - %w(linear linear), - %w(ease ease), - %w(ease-in ease-in), - %w(ease-in-out ease-in-out), - %w(ease-out ease-out), - %w(step-start step-start), - %w(step-end step-end) ], - 'color_value' => [ - %w(transparent transparent), - %w(currentColor currentColor), - %w(rgb() rgba()), - %w(hsl() hsla()), - %w(rgba() rgba()), - %w(hsla() hsla()) ]} + # The specification a page documents, cut down to the part of CSS it + # covers: "CSS Grid Layout Module Level 2" is the Grid Layout pages. + def specification + title = context[:generator].specification_titles(page.browser_compat, page.spec_urls).first + return if title.blank? + + title = title.dup + title.remove! 'CSS ' + title.remove! ' Module' + title.remove! %r{ Level \d+\z} + title.remove! %r{\(.*\)} + title.sub! ' and ', ' & ' + title.squish! - def additional_entries - ADDITIONAL_ENTRIES[slug] || [] + _, type = TYPE_BY_SPEC.find { |spec, _| title.include?(spec) } + type || (title.match?(/\ALevel \d+\z/) ? nil : title.presence) end - def include_default_entry? - return true unless warning = at_css('.warning').try(:content) - !warning.include?('CSS Flexible Box') && !warning.include?('replaced in newer drafts') + def page + context[:page] end end end diff --git a/lib/docs/filters/dom/clean_html.rb b/lib/docs/filters/dom/clean_html.rb deleted file mode 100644 index d670446845..0000000000 --- a/lib/docs/filters/dom/clean_html.rb +++ /dev/null @@ -1,44 +0,0 @@ -module Docs - class Dom - class CleanHtmlFilter < Filter - def call - root_page? ? root : other - doc - end - - def root - end - - def other - css('h1 + br').remove - - # Bug fix: HTMLElement.offsetWidth - css('#offsetContainer .comment').remove - - css('section', 'font').each do |node| - node.before(node.children).remove - end - - # Bug fix: CompositionEvent, DataTransfer, etc. - if (div = at_css('div[style]')) && div['style'].include?('border: solid #ddd 2px') - div.remove - end - - # Remove double heading on SVG pages - if slug.start_with? 'SVG' - at_css('h2:first-child').try :remove - end - - # Remove
wrapping .overheadIndicator - css('div > .overheadIndicator:first-child:last-child', 'div > .blockIndicator:first-child:last-child').each do |node| - node.parent.replace(node) - end - - css('.syntaxbox > pre:first-child:last-child').each do |node| - node['class'] = 'syntaxbox' - node.parent.before(node).remove - end - end - end - end -end diff --git a/lib/docs/filters/dom/entries.rb b/lib/docs/filters/dom/entries.rb index f65871e127..682133741f 100644 --- a/lib/docs/filters/dom/entries.rb +++ b/lib/docs/filters/dom/entries.rb @@ -220,11 +220,9 @@ def get_type return value if name =~ key end - if spec = css('.standard-table').last - spec = spec.content - TYPE_BY_SPEC.each_pair do |key, value| - return value if spec.include?(key) - end + spec = specification_titles.join(' ') + TYPE_BY_SPEC.each_pair do |key, value| + return value if spec.include?(key) end links_text = css('a').map(&:content).join @@ -239,47 +237,14 @@ def get_type end end - SKIP_CONTENT = [ - 'not on a standards track', - 'removed from the Web', - 'not on a current W3C standards track', - 'This feature is not built into all browsers', - 'not currently supported in any browser' - ] - - def include_default_entry? - return true if type == 'Console' - return true unless node = doc.at_css('.overheadIndicator, .blockIndicator') - node = node.parent while node.parent != doc - return true if node.previous_element.try(:name).in?(%w(h2 h3)) - content = node.content - SKIP_CONTENT.none? { |str| content.include?(str) } + # The specifications the page documents, which used to be read off the + # table MDN rendered into it. + def specification_titles + @specification_titles ||= context[:generator].specification_titles(page.browser_compat, page.spec_urls) end - def additional_entries - entries = [] - - if slug == 'history' || slug == 'XMLHttpRequest' - css('dt a[href^="https://developer.mozilla.org"]').each do |node| - next if node.parent.at_css('.obsolete') || node.content.include?('moz') - name = node.content.sub('History', 'history') - id = node.parent['id'] = name.parameterize - entries << [name, id] - end - end - - if slug == 'XMLHttpRequest' - css('h2[id="Methods_2"] ~ h3').each do |node| - break if node.content == 'Non-standard methods' - entries << ["#{name}.#{node.content}", node['id']] - end - end - - if slug == 'History_API' - entries << ['history.pushState()', 'The_pushState()_method'] - end - - entries + def page + context[:page] end end end diff --git a/lib/docs/filters/javascript/clean_html.rb b/lib/docs/filters/javascript/clean_html.rb deleted file mode 100644 index 1b450470fc..0000000000 --- a/lib/docs/filters/javascript/clean_html.rb +++ /dev/null @@ -1,31 +0,0 @@ -module Docs - class Javascript - class CleanHtmlFilter < Filter - def call - root_page? ? root : other - doc - end - - def root - end - - def other - # Remove "style" attribute - css('.inheritsbox', '.overheadIndicator', '.blockIndicator').each do |node| - node.remove_attribute 'style' - end - - # Remove
wrapping .overheadIndicator - css('div > .overheadIndicator:first-child:last-child', 'div > .blockIndicator:first-child:last-child').each do |node| - node.parent.replace(node) - end - - css('.baseline-indicator').each do |node| - if node.next.text == '> ' - node.next.remove - end - end - end - end - end -end diff --git a/lib/docs/filters/javascript/entries.rb b/lib/docs/filters/javascript/entries.rb index 06e4bbff67..e895b98bcf 100644 --- a/lib/docs/filters/javascript/entries.rb +++ b/lib/docs/filters/javascript/entries.rb @@ -3,14 +3,13 @@ class Javascript class EntriesFilter < Docs::EntriesFilter TYPES = %w(Array ArrayBuffer Atomics Boolean DataView Date Function Generator Intl JSON Map Math Number Object PluralRules Promise Reflect RegExp - Set SharedArrayBuffer SIMD String Symbol TypedArray WeakMap WeakSet) + Set SharedArrayBuffer String Symbol TypedArray WeakMap WeakSet) INTL_OBJECTS = %w(Collator DateTimeFormat NumberFormat) def get_name if slug.start_with? 'Global_Objects/' name, method, *rest = *slug.sub('Global_Objects/', '').split('/') name.prepend 'Intl.' if INTL_OBJECTS.include?(name) - name.prepend 'SIMD.' if html.include?("SIMD.#{name}") if method unless method == method.upcase || method == 'NaN' @@ -21,7 +20,7 @@ def get_name if name.exclude?('.prototype') path = name.split('.') - if ((node = at_css('.syntaxbox') || at_css('code')) && node.content =~ /(?:\s|\A)[a-z\_][a-zA-Z\_]+\.#{path.last}/) || + if ((node = at_css('code')) && node.content =~ /(?:\s|\A)[a-z\_][a-zA-Z\_]+\.#{path.last}/) || ((node = at_css('.standard-table')) && node.content =~ /\.prototype[\[\.]#{path.last}/) path[-2] = path[-2][0].downcase + path[-2][1..-1] name = path.join('.') @@ -59,8 +58,6 @@ def get_type 'Errors' elsif INTL_OBJECTS.include?(object) 'Intl' - elsif name.start_with?('SIMD') - 'SIMD' elsif method || TYPES.include?(object) object else @@ -70,34 +67,6 @@ def get_type 'Miscellaneous' end end - - def additional_entries - return [] unless root_page? - entries = [] - - %w(arithmetic assignment bitwise comparison logical).each do |s| - css("a[href^='operators/#{s}_operators#']").each do |node| - name = CGI::unescapeHTML(node.content.strip) - name.remove! %r{[a-zA-Z]} - name.strip! - entries << [name, node['href'], 'Operators'] - end - end - - entries.uniq - end - - def include_default_entry? - node = doc.at_css '.blockIndicator, .warning' - - # Can't use :first-child because #doc is a DocumentFragment - return true unless node && node.parent == doc && !node.previous_element - - !node.content.include?('not on a standards track') && - !node.content.include?('removed from the Web') && - !node.content.include?('SpiderMonkey-specific feature, and will be removed') && - !node.content.include?('could be removed at any time') - end end end end diff --git a/lib/docs/filters/mdn_git/clean_html.rb b/lib/docs/filters/mdn_git/clean_html.rb new file mode 100644 index 0000000000..3ed7bb3238 --- /dev/null +++ b/lib/docs/filters/mdn_git/clean_html.rb @@ -0,0 +1,155 @@ +module Docs + class MdnGit + # Turns the HTML of the rendered markdown into the HTML MDN publishes: the + # definition lists and the note cards its authors write in markdown, and + # the ids its pages link their sections by. + class CleanHtmlFilter < Filter + # MDN's h1 carries the title of the page, which nothing links to. + SECTIONS = 'h2, h3, h4, h5, h6, dt' + + DEFINITION = ': ' + + # https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#alerts + ALERTS = { + 'NOTE' => ['note', 'Note'], + 'WARNING' => ['warning', 'Warning'], + 'IMPORTANT' => ['note', 'Important'], + 'TIP' => ['note', 'Tip'], + 'CAUTION' => ['warning', 'Caution'] + } + + ALERT = /\A\s*\[!([A-Z]+)\]\s*/ + + def call + definition_lists + note_cards + code_blocks + section_ids + tables + empty_paragraphs + doc + end + + private + + # MDN writes its definition lists as lists whose every item is a term + # followed by a one-item list holding the definition, prefixed with a + # colon. See markdown/m2h/handlers/dl.ts in https://github.com/mdn/yari. + def definition_lists + # Innermost first, so that a nested list is a
by the time the one + # around it is looked at and can no longer be taken for a definition. + css('ul').to_a.reverse_each do |list| + next unless definition_list?(list) + list.name = 'dl' + list.element_children.each { |item| replace_with_definition(item) } + end + end + + def definition_list?(list) + items = list.element_children + items.any? && items.all? do |item| + definitions = item.element_children.last + next false unless definitions && definitions.name == 'ul' && item.children.size > 1 + next false unless definitions.element_children.size == 1 + definition?(definitions.element_children.first) + end + end + + def definition?(item) + leading_text(item).to_s.lstrip.start_with?(DEFINITION) + end + + def replace_with_definition(item) + definitions = item.element_children.last + definition = definitions.element_children.first + + term = Nokogiri::XML::Node.new('dt', item.document) + item.children.to_a.take_while { |child| child != definitions }.each { |child| term << child } + + described = Nokogiri::XML::Node.new('dd', item.document) + definition.children.to_a.each { |child| described << child } + remove_definition_prefix described + + item.add_previous_sibling term + item.replace described + end + + def remove_definition_prefix(node) + return unless (text = leading_text_node(node)) + text.content = text.content.lstrip.delete_prefix(DEFINITION) + end + + def leading_text(node) + leading_text_node(node)&.content + end + + def leading_text_node(node) + node.xpath('.//text()').find { |text| text.content.present? && !text.content.strip.empty? } + end + + # GitHub's alerts, which MDN renders as its note cards. + def note_cards + css('blockquote').each do |quote| + next unless (text = leading_text_node(quote)) + next unless (alert = text.content.match(ALERT)) + next unless (name, title = ALERTS[alert[1]]) + + text.content = text.content.sub(ALERT, '') + (quote.at_css('p') || quote).prepend_child %(#{title}: ) + + quote.name = 'div' + quote['class'] = "notecard #{name}" + end + end + + def code_blocks + css('pre > code').each do |code| + pre = code.parent + language = code['class'].to_s[/language-(\S+)/, 1] + pre['data-language'] = language if language + pre.content = code.content + end + end + + # MDN gives every heading and every term an id, which is what the links + # to a section of a page point at. See kumascript/src/api/util.ts in + # https://github.com/mdn/yari. + def section_ids + taken = Set.new + + css(SECTIONS).each do |node| + term = node.name == 'dt' + id = node['id']&.downcase + + if id.blank? + # A term can be followed by a badge, which isn't part of its name. + text = term ? node.element_children.first&.content || node.content : node.content + id = MdnContent.slugify(text) + id = "#{id}_#{taken.size}" if id.blank? + id = (2..).lazy.map { |count| "#{id}_#{count}" }.find { |candidate| !taken.include?(candidate) } if taken.include?(id) + end + + taken << id + node['id'] = id + + next unless term + first = node.element_children.first + next if first.nil? || first.name == 'a' || first.at_css('a') + first.replace %(#{first.to_html}) + end + end + + def tables + css('table').each do |table| + table.before %(
) + table.previous_element << table + end + end + + # Left behind by the macros that don't render to anything, e.g. {{JSRef}}. + def empty_paragraphs + css('p').each { |node| node.remove if node.content.strip.empty? && node.element_children.empty? } + end + end + end +end diff --git a/lib/docs/filters/mdn_git/macros.rb b/lib/docs/filters/mdn_git/macros.rb new file mode 100644 index 0000000000..220f65f208 --- /dev/null +++ b/lib/docs/filters/mdn_git/macros.rb @@ -0,0 +1,45 @@ +module Docs + class MdnGit + # Expands the KumaScript macro calls MDN leaves in its markdown. + class MacrosFilter < Filter + # A macro can expand into another one: the tables MDN builds out of its + # own data are written with the same cross-references as its prose. Two + # rounds are enough for those, the third is a stop. + PASSES = 3 + + def call + macros = MdnContent::Macros.new context[:page], context[:pages], context[:generator] + PASSES.times { break unless expand(macros) } + doc + end + + private + + def expand(macros) + expanded = false + + # A macro call in a code block is quoted rather than called, as on the + # page documenting the "Regular expression syntax error". The calls in + # the code that runs through the prose are expanded like any other. + xpath('.//text()[not(ancestor::pre)]').each do |node| + next unless macros.expand?(node.content) + html = macros.expand(node.content) + node = paragraph_of(node) if macros.block?(html) + html.empty? ? node.remove : node.replace(html) + expanded = true + end + + expanded + end + + # The paragraph a macro that renders to a block of its own sits in, which + # it replaces instead of nesting itself into. + def paragraph_of(node) + parent = node.parent + return node unless parent&.name == 'p' + return node unless parent.children.all? { |child| child == node || (child.text? && child.content.strip.empty?) } + parent + end + end + end +end diff --git a/lib/docs/mdn_content/css.rb b/lib/docs/mdn_content/css.rb new file mode 100644 index 0000000000..cc14197fe3 --- /dev/null +++ b/lib/docs/mdn_content/css.rb @@ -0,0 +1,185 @@ +require 'cgi' + +module Docs + module MdnContent + # The two sections MDN generates out of mdn-data on the pages of the CSS + # reference: the formal syntax of a property, type, function or at-rule, + # and the table of its characteristics below it. + # + # MDN expands and aligns the syntax with css-tree; this prints the same + # definitions, one alternative per line, without the alignment and without + # linking every multiplier to the page explaining it. See + # kumascript/src/lib/css-syntax.ts in https://github.com/mdn/yari. + class Css + REFERENCE = '/en-US/docs/Web/CSS/Reference' + + # How deep the definitions of the types a syntax refers to are followed. + # MDN stops when it runs out of definitions; the limit is a guard against + # the types that refer back to each other. + DEPTH = 10 + + # The rows of the characteristics table, and the order MDN puts them in. + CHARACTERISTICS = { + 'initial' => 'Initial value', + 'appliesto' => 'Applies to', + 'inherited' => 'Inherited', + 'percentages' => 'Percentages', + 'computed' => 'Computed value', + 'animationType' => 'Animation type' + }.freeze + + # A reference to the definition of a type, e.g. or + # . The references to a property are quoted — + # <'animation-name'> — which is why they don't match. + TYPE = /<([\w-]+(?:\(\))?)(?:\s*\[[^\]]*\])?>/ + + # The same reference to a property, once escaped. + PROPERTY = /<'([\w-]+)'>/ + + # The combinators a definition is broken into lines on, outside of any + # brackets. + COMBINATOR = /\s(\|\||&&|\||\s)\s/ + + def initialize(data) + @data = data + end + + # The formal syntax of the thing a page documents, followed by the + # definitions of the types it's made of. + def syntax(page) + name, syntax = definition(page) + return if syntax.blank? + + blocks = [[name, syntax]] + seen = [name] + pending = [syntax] + + DEPTH.times do + pending = pending.flat_map { |value| references(value) }.uniq + pending -= seen + pending = pending.filter_map do |reference| + next unless (value = @data.css_syntax(reference)) + seen << reference + blocks << ["<#{reference}>", value] + value + end + break if pending.empty? + end + + %(
#{blocks.map { |block| render(*block) }.join("\n")}
) + end + + def raw_syntax(syntax) + %(
#{escape syntax.delete('`')}
) + end + + # The table of the characteristics of a property or an at-rule + # descriptor, e.g. whether it's inherited and how it animates. + def info(page, name = nil, at_rule = nil) + return unless (entry = characteristics(page, name, at_rule)) + + rows = CHARACTERISTICS.filter_map do |key, label| + value = entry[key] + next if value.nil? || value == '' || (key == 'percentages' && value == 'no') + %(#{label}#{value_html value}) + end + + return if rows.empty? + %(#{rows.join}
) + end + + private + + # The name a page documents and the syntax of its definition, both of + # which depend on what kind of page it is. + def definition(page) + name = page.slug.split('/').last.downcase + + case page.page_type + when 'css-property', 'css-shorthand-property' + [name, @data.css('properties', name)&.fetch('syntax', nil)] + when 'css-type' + name = name.delete_suffix('_value') + [+"<#{name}>", @data.css_syntax(name) || @data.css('types', name)&.fetch('syntax', nil)] + when 'css-function' + [+"<#{name}()>", @data.css_syntax("#{name}()") || @data.css('functions', "#{name}()")&.fetch('syntax', nil)] + when 'css-at-rule' + [+"@#{name}", @data.css('at-rules', "@#{name}")&.fetch('syntax', nil)] + when 'css-at-rule-descriptor' + [name, descriptor(page, name)&.fetch('syntax', nil)] + else + [name, nil] + end + end + + def characteristics(page, name, at_rule) + name ||= page.slug.split('/').last.downcase + + if at_rule.present? || page.page_type == 'css-at-rule-descriptor' + descriptor page, name, at_rule + else + @data.css 'properties', name + end + end + + # An at-rule descriptor is documented one level below its at-rule. + def descriptor(page, name, at_rule = nil) + at_rule ||= "@#{page.slug.split('/')[-2].to_s.downcase}" + at_rule = "@#{at_rule}" unless at_rule.start_with?('@') + @data.css('at-rules', at_rule)&.dig('descriptors', name) + end + + def render(name, syntax) + lines = split(syntax).map { |line| " #{markup line}" } + "#{escape name} = \n#{lines.join("\n")}\n" + end + + # Breaks a definition on the combinators that aren't inside brackets, the + # way MDN lays it out. + def split(syntax) + lines = [''] + depth = 0 + + syntax.split(/(?<=[\s\[\]])|(?=[\[\]])/).each do |token| + depth += token.count('[') - token.count(']') + lines.last << token + lines << '' if depth.zero? && token.match?(/\A\s*(\|\||&&|\|)\s*\z/) + end + + lines.map(&:strip).reject(&:empty?) + end + + # The references to a property are the only ones worth a link: the types + # are spelled out right below. + def markup(line) + escape(line).gsub(PROPERTY) { %(<'#{$1}'>) } + end + + def references(syntax) + syntax.scan(TYPE).flatten + end + + # A shorthand takes its value from every property it stands for; the + # rest read as a string, or as one of the values mdn-data enumerates. + def value_html(value) + case value + when true then 'yes' + when false then 'no' + when Array + items = value.map { |name| "
  • #{link name}
    • " } + %(as each of the properties of the shorthand:
      #{items.join}
    ) + else + escape(@data.css_string(value) || value) + end + end + + def link(name) + %(#{escape name}) + end + + def escape(text) + CGI.escape_html text.to_s + end + end + end +end diff --git a/lib/docs/mdn_content/data.rb b/lib/docs/mdn_content/data.rb new file mode 100644 index 0000000000..9c58851c74 --- /dev/null +++ b/lib/docs/mdn_content/data.rb @@ -0,0 +1,185 @@ +require 'json' + +module Docs + module MdnContent + # The data MDN pulls in when it builds a page: the browser compatibility + # tables (@mdn/browser-compat-data), the titles of the specifications + # (web-specs) and the Baseline status of the features (web-features). + # + # The packages are pruned down to what's actually needed when they're + # downloaded (see .prepare). Their unpruned form weighs 25MB of JSON, which + # the scraper would otherwise read in every one of its forked workers. + class Data + COMPAT = 'compat.json' + SPECS = 'specs.json' + BASELINE = 'baseline.json' + CSS = 'css.json' + + def initialize(directory) + @directory = directory + end + + # The compatibility data of a feature, e.g. "javascript.builtins.Array.map". + def compat(query) + query.split('.').inject(compat_data) do |data, key| + break nil unless data.is_a?(Hash) + data[key] + end + end + + def browsers + compat_data['browsers'] ||= {} + end + + # The date a browser version was released, when it's known. + def release_date(browser, version) + browsers.dig(browser, 'releases', version, 'release_date') + end + + # The specification a URL points into. The URLs in browser-compat-data + # point at a section of the editor's draft more often than not, which is + # why the nightly urls are matched as well. + def spec(url) + specs.find do |spec| + url.start_with?(spec['url']) || + Array(spec['nightly_urls']).any? { |nightly| url.start_with?(nightly) } + end + end + + # The Baseline status of the feature a compatibility query documents. + def baseline(query) + baseline_data[query] + end + + # What MDN knows about a CSS property, type, function or at-rule beyond + # its own prose: its formal syntax and the table of its characteristics. + def css(kind, name) + css_data.dig(kind, name) + end + + # The definition of a CSS value type, e.g. "single-animation". + def css_syntax(name) + css_data.dig('syntaxes', name) + end + + # The English of the enumerated values of mdn-data, e.g. "allElements". + def css_string(key) + css_data.dig('l10n', key) + end + + private + + def compat_data + @compat_data ||= read(COMPAT) + end + + def specs + @specs ||= read(SPECS) + end + + def baseline_data + @baseline_data ||= read(BASELINE) + end + + def css_data + @css_data ||= File.exist?(File.join(@directory, CSS)) ? read(CSS) : {} + end + + def read(name) + JSON.parse File.read(File.join(@directory, name)) + end + + class << self + # Turns the three packages unpacked into directory into the files the + # scraper reads, and removes them. Only the sections of the + # compatibility data the documentation queries are kept. + def prepare(directory, namespaces) + prepare_compat directory, namespaces + prepare_specs directory + prepare_baseline directory, namespaces + prepare_css directory if Dir.exist?(File.join(directory, 'mdn-data')) + FileUtils.rm_rf File.join(directory, 'mdn-data') + end + + private + + def prepare_compat(directory, namespaces) + data = read_package(directory, 'browser-compat-data', 'data.json') + write directory, COMPAT, data.slice('browsers', *namespaces) + end + + def prepare_specs(directory) + specs = read_package(directory, 'web-specs', 'index.json') + + # Only the series' nightly url of the current specification is worth + # keeping: an older version would shadow the one MDN links to. + specs = specs.map do |spec| + nightly_urls = [] + nightly_urls.push(spec.dig('nightly', 'url'), *spec.dig('nightly', 'alternateUrls')) + nightly_urls << spec.dig('series', 'nightlyUrl') if spec['shortname'] == spec.dig('series', 'currentSpecification') + { 'url' => spec['url'], 'title' => spec['title'], 'nightly_urls' => nightly_urls.compact.uniq } + end + + write directory, SPECS, specs + end + + def prepare_baseline(directory, namespaces) + data = read_package(directory, 'web-features', 'data.json') + sections = namespaces.map { |namespace| "#{namespace}." } + baseline = {} + + data['features'].each_value do |feature| + status = feature['status'] + next unless status + + Array(feature['compat_features']).each do |query| + next unless sections.any? { |section| query.start_with?(section) } + baseline[query] = status.slice('baseline', 'baseline_low_date', 'baseline_high_date') + end + end + + write directory, BASELINE, baseline + end + + # What the CSS documentation reads out of mdn-data: the formal syntax + # of every property, type, function and at-rule, the characteristics + # MDN tabulates below it, and the English of the values it names. + CSS_FIELDS = %w(syntax initial appliesto inherited percentages computed animationType) + + def prepare_css(directory) + css = {} + + %w(properties types functions at-rules).each do |file| + data = read_package(directory, 'mdn-data', "css/#{file}.json", keep: true) + css[file] = data.transform_values { |value| slice_css value } + end + + syntaxes = read_package(directory, 'mdn-data', 'css/syntaxes.json', keep: true) + css['syntaxes'] = syntaxes.transform_values { |value| value['syntax'] } + + strings = read_package(directory, 'mdn-data', 'l10n/css.json', keep: true) + css['l10n'] = strings.transform_values { |translations| translations['en-US'] }.compact + + write directory, CSS, css + end + + def slice_css(value) + sliced = value.slice(*CSS_FIELDS) + descriptors = value['descriptors'] + sliced['descriptors'] = descriptors.transform_values { |descriptor| descriptor.slice(*CSS_FIELDS) } if descriptors + sliced + end + + def read_package(directory, name, file, keep: false) + JSON.parse File.read(File.join(directory, name, file)) + ensure + FileUtils.rm_rf File.join(directory, name) unless keep + end + + def write(directory, name, data) + File.write File.join(directory, name), JSON.generate(data) + end + end + end + end +end diff --git a/lib/docs/mdn_content/generator.rb b/lib/docs/mdn_content/generator.rb new file mode 100644 index 0000000000..ac5b364c25 --- /dev/null +++ b/lib/docs/mdn_content/generator.rb @@ -0,0 +1,259 @@ +require 'cgi' +require 'docs/mdn_content/css' + +module Docs + module MdnContent + # Builds the parts of a page that MDN generates out of data rather than out + # of markdown: the Baseline indicator, the status note cards, and the + # specifications and browser compatibility tables. + class Generator + BROWSERS = { + 'Desktop' => { + 'chrome' => 'Chrome', + 'edge' => 'Edge', + 'firefox' => 'Firefox', + 'opera' => 'Opera', + 'safari' => 'Safari' + }, + 'Mobile' => { + 'chrome_android' => 'Chrome Android', + 'firefox_android' => 'Firefox for Android', + 'opera_android' => 'Opera Android', + 'safari_ios' => 'Safari on iOS', + 'samsunginternet_android' => 'Samsung Internet', + 'webview_android' => 'WebView Android', + 'webview_ios' => 'WebView on iOS' + }, + 'Server' => { + 'bun' => 'Bun', + 'deno' => 'Deno', + 'nodejs' => 'Node.js' + } + }.freeze + + # https://github.com/mdn/yari/blob/main/client/src/lit/baseline-indicator.ts + BASELINE = { + 'high' => ['high', 'Widely available', <<~TEXT], + This feature is well established and works across many devices and + browser versions. + TEXT + 'low' => ['low', 'Newly available', <<~TEXT], + This feature works across the latest devices and browser versions. + This feature might not work in older devices or browser versions. + TEXT + 'false' => ['not', 'Limited availability', <<~TEXT] + This feature is not Baseline because it does not work in some of the + most widely-used browsers. + TEXT + }.freeze + + STATUSES = { + 'deprecated' => ['deprecated', 'Deprecated', <<~TEXT], + This feature is no longer recommended. Though some browsers might + still support it, it may have already been removed from the relevant + web standards, may be in the process of being dropped, or may only be + kept for compatibility purposes. Avoid using it, and update existing + code if possible; see the compatibility + table at the bottom of this page to guide your decision. Be aware + that this feature may cease to work at any time. + TEXT + 'experimental' => ['experimental', 'Experimental', <<~TEXT], + This is an experimental technology
    Check the + Browser compatibility table + carefully before using this in production. + TEXT + 'non-standard' => ['nonstandard', 'Non-standard', <<~TEXT] + This feature is non-standard and is not on a standards track. Do not + use it on production sites facing the Web: it will not work for every + user. There may also be large incompatibilities between implementations + and the behavior may change in the future. + TEXT + }.freeze + + def initialize(data) + @data = data + end + + # The sections MDN builds out of mdn-data, on the CSS pages only. + def css + @css ||= Css.new(@data) + end + + # Everything MDN puts above the content of a page: its title, how widely + # available the feature is, and whether it should be used at all. + def header(page) + html = +%(

    #{escape page.title}

    ) + html << baseline(page.browser_compat.first).to_s + page.status.each { |status| html << note_card(*STATUSES[status]) if STATUSES.key?(status) } + html + end + + def note_card(name, title, text) + %(

    #{title}: #{text.squish}

    ) + end + + def baseline(query) + return unless query && (status = @data.baseline(query)) + name, title, text = BASELINE[status['baseline'].to_s] + return unless name + + text = "#{text.squish} It's been available across browsers since #{month status['baseline_low_date']}." if status['baseline_low_date'] + + <<~HTML.squish +
    +
    Baseline #{title}
     +

    #{text.squish}

    +
    + HTML + end + + # The sections of browser-compat-data MDN lists the server-side runtimes + # for. The web APIs have data for them too, but no column. + SERVER_SECTIONS = %w(javascript webassembly) + + # The browser compatibility table of one or more features, e.g. + # "javascript.builtins.Array.map". + def compat(queries) + features = queries.flat_map { |query| compat_features(query) } + return if features.empty? + + browsers = BROWSERS + browsers = browsers.except('Server') unless queries.any? { |query| SERVER_SECTIONS.include?(query.split('.').first) } + ids = browsers.each_value.flat_map(&:keys) + + rows = features.map do |name, support| + cells = ids.map { |id| support_cell id, support[id] } + %(#{escape name}#{cells.join}) + end + + <<~HTML + + + #{['', *browsers.map { |platform, names| %() }].join} + #{['', *browsers.each_value.flat_map(&:values).map { |name| "" }].join} + + #{rows.join} +
    #{platform}
    #{name}
    + HTML + end + + # The specifications a feature is defined in. They're looked up in + # browser-compat-data unless the page names them itself. + def specifications(queries, urls) + urls = specification_urls(queries, urls) + return if urls.empty? + + rows = urls.map do |url| + title = specification_title(url) + fragment = url[/#(.+)\z/, 1] + title = %(#{escape title}
    # #{escape fragment}) if fragment + %(#{title}) + end + + <<~HTML + + + #{rows.join} +
    Specification
    + HTML + end + + # The names of the specifications a feature is defined in, which is how + # the entries filters tell one part of a documentation from another. + def specification_titles(queries, urls) + specification_urls(queries, urls).map { |url| specification_title url } + end + + private + + UNKNOWN_SPECIFICATION = 'Unknown specification' + + def specification_urls(queries, urls) + urls = queries.flat_map { |query| spec_urls @data.compat(query) } if urls.empty? + urls.uniq + end + + def specification_title(url) + @data.spec(url)&.fetch('title') || UNKNOWN_SPECIFICATION + end + + # The feature itself, when it has compatibility data of its own, followed + # by its subfeatures. A subfeature can go by the name of the feature — + # the constructor of Intl.Locale is javascript.builtins.Intl.Locale.Locale + # — hence the pairs rather than a hash. + def compat_features(query) + return [] unless (data = @data.compat(query)) + + features = [] + features << [query.split('.').last, data] if data.key?('__compat') + data.each { |key, value| features << [key, value] if value.is_a?(Hash) && value.key?('__compat') } + features.map { |name, feature| [name, feature['__compat']['support'] || {}] } + end + + # The first statement is the current one; the ones that follow it are the + # versions the feature was supported in before, under a prefix or a flag. + def support_cell(browser, support) + statements = Array.wrap(support) + return %(
    ?
    ) if statements.empty? + + versions = statements.map { |statement| support_version browser, statement } + %(#{versions.join}) + end + + def support_level(statement) + return 'no' if statement['version_added'] == false || statement['version_removed'] + return 'preview' if statement['version_added'] == 'preview' + return 'unknown' if statement['version_added'].nil? + return 'partial' if statement['partial_implementation'] || statement['flags'] + 'yes' + end + + def support_version(browser, statement) + added = statement['version_added'] + version = case added + when true then 'Yes' + when false then 'No' + when nil then '?' + else added + end + + version = "#{version}–#{statement['version_removed']}" if statement['version_removed'].is_a?(String) + + if added.is_a?(String) && (date = @data.release_date(browser, added)) + version = %(#{escape version}) + else + version = escape version + end + + notes = Array.wrap(statement['notes']) + return "
    #{version}
    " if notes.empty? + %(
    #{version}#{notes.join('
    ')}
    ) + end + + # The specification urls of a feature, or of its subfeatures when it has + # none of its own. + def spec_urls(data) + return [] unless data.is_a?(Hash) + return Array.wrap(data.dig('__compat', 'spec_url')) if data.key?('__compat') + data.each_value.flat_map { |value| spec_urls value } + end + + def month(date) + Date.parse(date).strftime('%B %Y') + rescue ArgumentError, TypeError + date + end + + def escape(text) + CGI.escape_html text.to_s + end + + # The urls of browser-compat-data point at the fragment the browsers gave + # a section of a specification, which is neither always ascii nor always + # escaped — %symbol.iterator% is a fragment, not an escape. + def escape_url(url) + url.gsub(/%(?![0-9A-Fa-f]{2})|[^\x21-\x7E]/) { |char| char.bytes.map { |byte| format('%%%02X', byte) }.join } + end + end + end +end diff --git a/lib/docs/mdn_content/macros.rb b/lib/docs/mdn_content/macros.rb new file mode 100644 index 0000000000..ad953aa852 --- /dev/null +++ b/lib/docs/mdn_content/macros.rb @@ -0,0 +1,408 @@ +require 'cgi' +require 'strscan' + +module Docs + module MdnContent + # Expands the KumaScript macro calls MDN leaves in its markdown, e.g. + # {{jsxref("Array")}} or {{Compat}}. Only the macros the JavaScript + # reference uses are implemented; the others are dropped. + # + # See https://github.com/mdn/yari/tree/main/kumascript/macros. + class Macros + DOCS = '/en-US/docs' + JAVASCRIPT = 'Web/JavaScript/Reference' + CSS = 'Web/CSS' + GLOBAL_OBJECTS = 'Global_Objects' + + # Where the CSS reference files its pages, which is what the names a + # cssxref goes by have to be looked for under. + CSS_SECTIONS = ['', 'Properties/', 'Values/', 'Selectors/', 'At-rules/'] + + # The macros whose expansion replaces the paragraph holding them rather + # than sitting inside it. + BLOCK = /\A<(?:table|div|details|pre|[uo]l|h\d)[\s>]/ + + CALL = /\{\{\s*([\w.-]+)/ + + def initialize(page, pages, generator) + @page = page + @pages = pages + @generator = generator + end + + def expand?(text) + text.include?('{{') + end + + # Replaces the macro calls in a string of text with their expansion, + # which is HTML — hence the escaping of everything around them. Anything + # that doesn't parse as a call is left alone. + def expand(text) + scanner = StringScanner.new(text) + result = +'' + + until scanner.eos? + result << CGI.escape_html(scanner.scan_until(/(?=\{\{)/).to_s) + break if scanner.eos? + + if (call = parse(scanner)) + result << call_macro(*call).to_s + else + result << CGI.escape_html(scanner.getch) + end + end + + result << CGI.escape_html(scanner.rest) + end + + def block?(html) + html.match? BLOCK + end + + private + + def parse(scanner) + position = scanner.pos + return revert(scanner, position) unless scanner.skip(/\{\{\s*/) && (name = scanner.scan(/[\w.-]+/)) + + arguments = [] + + if scanner.skip(/\s*\(/) + until scanner.skip(/\s*\)/) + argument = + if scanner.scan(/\s*"((?:[^"\\]|\\.)*)"/) || + scanner.scan(/\s*'((?:[^'\\]|\\.)*)'/) || + scanner.scan(/\s*`([^`]*)`/) + scanner[1] + else + # An argument can be left out, as in {{rfc("7002",,"3.2")}}, + # in which case there's nothing to scan but a comma follows. + scanner.scan(/[^,)]+/) || ('' if scanner.match?(/,/)) + end + + # Anything else isn't a call, and would have the scanner stand + # still until the end of time. + return revert(scanner, position) if argument.nil? + + # The arguments are written the way they read, which for the names + # of the CSS types means <color> rather than . + arguments << CGI.unescape_html(argument.strip) + scanner.skip(/\s*,/) + end + end + + return revert(scanner, position) unless scanner.skip(/\s*\}\}/) + [name, arguments] + end + + def revert(scanner, position) + scanner.pos = position + nil + end + + def call_macro(name, arguments) + name = name.downcase.tr('-', '_') + return unless MACROS.include?(name) + send(name, *arguments) + rescue ArgumentError + nil + end + + # + # Cross-references + # + + def jsxref(api, display = nil, anchor = nil, plain = nil) + slug = api.sub('()', '').sub('.prototype.', '.') + slug = slug.sub('.', '/') if api.include?('.') && !api.include?('/') + content = content_for(display || api, plain) + + if documents?(JAVASCRIPT) + slug = "#{GLOBAL_OBJECTS}/#{slug}" if !@pages.include?(slug) && @pages.include?("#{GLOBAL_OBJECTS}/#{slug}") + return content unless @pages.include?(slug) + end + + link "#{DOCS}/#{JAVASCRIPT}/#{slug}#{fragment anchor}", content + end + + def domxref(api, display = nil, anchor = nil, plain = nil) + slug = api.tr(' ', '_').remove('()').gsub('.prototype.', '.').tr('.', '/').sub(/\A./, &:upcase) + display = display.presence || api + display = "#{display}.#{anchor}" if anchor.present? + link "#{DOCS}/Web/API/#{slug}#{fragment anchor}", content_for(display, plain) + end + + def glossary(term, display = nil, plain = nil) + link "#{DOCS}/Glossary/#{term.tr(' ', '_')}", content_for(display || term, plain || '1') + end + + # The CSS reference files a name under one of its sections and spells it + # the way the section does — the page of the "length" type is + # Values/length and goes by . Outside the CSS documentation, the + # link is the one MDN redirects from. + def cssxref(name, display = nil, anchor = nil) + page = css_page(name) if documents?(CSS) + display = display.presence || page&.short_title || name + slug = page ? page.slug : css_name(name) + link "#{DOCS}/#{CSS}#{'/Reference' if page}/#{slug}#{fragment anchor}", content_for(display, nil) + end + + def css_page(name) + name = css_name(name) + CSS_SECTIONS.each do |section| + page = @pages["#{section}#{name}"] || @pages["#{section}#{name}_value"] + return page if page + end + nil + end + + # A type is referred to as and a function as calc(), neither of + # which is part of the name of their page. + def css_name(name) + name.remove('()').remove(/\A<|>\z/) + end + + # Whether the documentation being built is the one a cross-reference + # points into, rather than one linking out to it. + def documents?(prefix) + @pages.prefix.start_with?(prefix) + end + + def htmlelement(name, display = nil, anchor = nil) + name = name.downcase + display = display.presence || "<#{name}>" + link "#{DOCS}/Web/HTML/Element/#{name}#{fragment anchor}", content_for(display, nil) + end + + def httpheader(name, display = nil, anchor = nil, plain = nil) + display = display.presence || name + display = "#{display}.#{anchor}" if anchor.present? + link "#{DOCS}/Web/HTTP/Headers/#{name}#{fragment anchor}", content_for(display, plain) + end + + def svgelement(name, *) + link "#{DOCS}/Web/SVG/Reference/Element/#{name}", content_for("<#{name}>", nil) + end + + def svgattr(name, *) + link "#{DOCS}/Web/SVG/Reference/Attribute/#{name}", content_for(name, nil) + end + + def mathmlelement(name, display = nil, anchor = nil) + display = display.presence || "<#{name}>" + link "#{DOCS}/Web/MathML/Reference/Element/#{name}#{fragment anchor}", content_for(display, nil) + end + + def httpmethod(name, display = nil, anchor = nil, plain = nil) + link "#{DOCS}/Web/HTTP/Reference/Methods/#{name}#{fragment anchor}", content_for(display.presence || name, plain) + end + + def csp(directive, *) + link "#{DOCS}/Web/HTTP/Reference/Headers/Content-Security-Policy/#{directive}", content_for(directive, nil) + end + + def httpstatus(code, display = nil, anchor = nil, plain = nil) + link "#{DOCS}/Web/HTTP/Reference/Status/#{code}#{fragment anchor}", content_for(display.presence || code, plain) + end + + def webextapiref(name, display = nil, *) + slug = name.tr('.', '/') + link "#{DOCS}/Mozilla/Add-ons/WebExtensions/API/#{slug}", content_for(display.presence || name, nil) + end + + def rfc(number, title = nil, section = nil) + url = "https://datatracker.ietf.org/doc/html/rfc#{number}" + text = +"RFC #{number}" + + if section.present? + url << "#section-#{section}" + text << ", section #{section}" + end + + text << ": #{title}" if title.present? + link url, escape(text) + end + + # + # Badges and note cards + # + + def optional_inline + %(Optional) + end + + def deprecated_inline + badge 'deprecated', 'Deprecated', 'Deprecated. Not for use in new websites.' + end + + def experimental_inline + badge 'experimental', 'Experimental', 'Experimental. Expect behavior to change in the future.' + end + + def non_standard_inline + badge 'nonStandard', 'Non-standard', 'Non-standard. Check cross-browser support before using.' + end + + def readonlyinline + %(Read only) + end + + def securecontext_inline + %(Secure context) + end + + def securecontext_header + @generator.note_card 'secure', 'Secure context', <<~TEXT + This feature is available only in secure + contexts (HTTPS), in some or all supporting browsers. + TEXT + end + + # https://github.com/mdn/yari/blob/main/kumascript/macros/AvailableInWorkers.ejs + WORKERS = "Web Workers" + SERVICE_WORKERS = "Service Workers" + + WORKER_SCOPES = { + nil => "This feature is available in #{WORKERS}.", + 'worker' => "This feature is only available in #{WORKERS}.", + 'window_and_worker_except_service' => "This feature is available in #{WORKERS}, except for #{SERVICE_WORKERS}.", + 'worker_except_service' => "This feature is only available in #{WORKERS}, except for #{SERVICE_WORKERS}.", + 'window_and_worker_except_shared' => %(This feature is available in #{WORKERS}, except for Shared Web Workers.), + 'window_and_dedicated' => %(This feature is available in Dedicated Web Workers.), + 'dedicated' => %(This feature is only available in Dedicated Web Workers.), + 'window_and_service' => "This feature is available in #{SERVICE_WORKERS}.", + 'service' => "This feature is only available in #{SERVICE_WORKERS}." + } + + def availableinworkers(scope = nil) + @generator.note_card 'note', 'Note', WORKER_SCOPES.fetch(scope.presence, WORKER_SCOPES[nil]) + end + + def seecompattable + @generator.note_card(*Generator::STATUSES['experimental']) + end + + def non_standard_header + @generator.note_card(*Generator::STATUSES['non-standard']) + end + + def deprecated_header + @generator.note_card(*Generator::STATUSES['deprecated']) + end + + # + # Generated sections + # + + def compat(query = nil) + @generator.compat(queries(query, @page.browser_compat)) + end + + def specifications(query = nil) + @generator.specifications(queries(query, @page.browser_compat), @page.spec_urls) + end + + def interactiveexample(*) + %(

    Try it

    ) + end + + # The index of a landing page. MDN nests it as deep as it's asked to; + # one level is all its own pages ever ask for. + def listsubpages(path = nil, _depth = nil, reverse = nil, ordered = nil) + children = children_of(path) + return if children.empty? + + children = children.reverse if reverse.to_s == '1' + items = children.map { |child| %(
  • #{link "#{DOCS}/#{@pages.prefix}/#{child.slug}", escape(child.title)}
    • ) } + tag = ordered.to_s == '1' ? 'ol' : 'ul' + "<#{tag}>#{items.join}" + end + + def csssyntax(*) + @generator.css.syntax @page + end + + def csssyntaxraw(syntax) + @generator.css.raw_syntax syntax + end + + def cssinfo(name = nil, at_rule = nil) + @generator.css.info @page, name.presence, at_rule.presence + end + + def js_property_attributes(writable, enumerable, configurable) + rows = { 'Writable' => writable, 'Enumerable' => enumerable, 'Configurable' => configurable } + rows = rows.map { |name, value| "#{name}#{value.to_s == '1' ? 'yes' : 'no'}" } + + <<~HTML + + + #{rows.join} +
    Property attributes of #{escape @page.title}
    + HTML + end + + # MDN follows every link of this index with the opening paragraph of the + # page it points at. Reading all of those pages to summarize them is a + # lot of work for the handful of landing pages asking for it, so the + # index is the same one ListSubPages renders. + alias_method :subpageswithsummaries, :listsubpages + + # What a page carries none of into DevDocs: the sidebars, the live + # samples, the inheritance diagrams and the links to the next page. + DROPPED = %w( + addonsidebar apiref cssref css_ref defaultapisidebar htmlsidebar jsref + jssidebar mathmlref svgref webextsidebar + embedghlivesample embedlivesample embedyoutube livesamplelink + inheritancediagram previousnext previous next previousmenunext + previousmenu nextmenu listgroups apilistalpha) + + DROPPED.each { |name| define_method(name) { |*| nil } } + + MACROS = (%w( + jsxref domxref glossary cssxref htmlelement httpheader httpmethod rfc + svgelement svgattr mathmlelement csp httpstatus webextapiref + optional_inline deprecated_inline experimental_inline non_standard_inline + readonlyinline securecontext_inline + seecompattable non_standard_header deprecated_header securecontext_header + availableinworkers + compat specifications interactiveexample js_property_attributes + csssyntax csssyntaxraw cssinfo + listsubpages subpageswithsummaries) + DROPPED).to_set.freeze + + # + # Helpers + # + + def children_of(path) + @pages.children path.to_s.sub(%r{\A/[^/]+/docs/}, '').sub(%r{\A#{Regexp.escape @pages.prefix}/?}, '') + end + + def queries(query, default) + query.present? ? query.split(',').map(&:strip) : default + end + + def content_for(text, plain) + text = escape(text) + plain.present? ? text : "#{text}" + end + + def escape(text) + CGI.escape_html text.to_s + end + + def fragment(anchor) + "##{anchor.delete_prefix('#')}" if anchor.present? + end + + def link(url, content) + %(#{content}) + end + + # DevDocs styles the class of the inner element, MDN the icon around it. + def badge(name, title, description) + %(#{title}) + end + end + end +end diff --git a/lib/docs/mdn_content/markdown.rb b/lib/docs/mdn_content/markdown.rb new file mode 100644 index 0000000000..52d33ddea0 --- /dev/null +++ b/lib/docs/mdn_content/markdown.rb @@ -0,0 +1,88 @@ +require 'cgi' +require 'kramdown' +require 'kramdown-parser-gfm' + +module Docs + module MdnContent + # Renders the markdown of an MDN page. MDN renders it with remark and the + # GitHub extensions, which kramdown's GFM parser comes closest to; the + # constructs that are specific to MDN — its definition lists, note cards + # and code fences — are turned into what MDN makes of them afterwards, in + # Javascript::CleanHtmlFilter. + module Markdown + # MDN's markdown is not typeset: replacing "..." with an ellipsis or "--" + # with a dash would rewrite the code that's inline in the prose, starting + # with the names of the "try...catch" and "for...of" pages. + SYMBOLS = { + hellip: '...', + mdash: '---', + ndash: '--', + laquo: '<<', + raquo: '>>', + laquo_space: '<< ', + raquo_space: ' >>' + } + + OPTIONS = { + input: 'GFM', + auto_ids: false, + hard_wrap: false, + syntax_highlighter: nil, + smart_quotes: %w(apos apos quot quot), + typographic_symbols: SYMBOLS + } + + # The lint switch MDN appends to the name of a language, and the names it + # gives the blocks that aren't code to begin with. + NOLINT = '-nolint' + NOT_A_LANGUAGE = %w(plain none text unix) + + OPENING_FENCE = /\A([ ]{0,3})([~`]{3,})[ \t]*(\S[^\n]*)?\n?\z/ + + # A macro call and what it's held out of the parser's reach as, which is + # a word markdown has no reason to touch. + MACRO = /\{\{.*?\}\}/m + TOKEN = /kumascript(\d+)kumascript/ + + def self.render(source) + macros = [] + source = normalize_fences(source).gsub(MACRO) { |macro| "kumascript#{macros.push(macro).size - 1}kumascript" } + + # kramdown edits the options it's handed, hence the copy. + html = Kramdown::Document.new(source, OPTIONS.deep_dup).to_html + html.gsub(TOKEN) { CGI.escape_html macros[$1.to_i] } + end + + # A macro call means nothing to markdown, which is free to read the + # punctuation in its arguments as emphasis and tear it in half — as in + # {{jsxref("Statements/function*", "function*")}}. MDN puts its calls out + # of reach for the same reason; see markdown/utils/index.ts in + # https://github.com/mdn/yari. + # + # Reduces the info string of the code fences to their language. MDN tags + # them with the role they play in the page as well — "js example-bad", + # "js interactive-example" — which kramdown doesn't expect, and which + # makes it miss the fence altogether. + def self.normalize_fences(source) + closing = nil + + source.lines.map! do |line| + if closing + closing = nil if line.match?(closing) + line + elsif (fence = line.match(OPENING_FENCE)) + closing = /\A[ ]{0,3}#{fence[2]}#{fence[2][0]}*[ \t]*\n?\z/ + "#{fence[1]}#{fence[2]}#{language(fence[3])}\n" + else + line + end + end.join + end + + def self.language(info) + name = info.to_s[/\A\S+/].to_s.delete_suffix(NOLINT) + NOT_A_LANGUAGE.include?(name) ? '' : name + end + end + end +end diff --git a/lib/docs/mdn_content/pages.rb b/lib/docs/mdn_content/pages.rb new file mode 100644 index 0000000000..4deb91372e --- /dev/null +++ b/lib/docs/mdn_content/pages.rb @@ -0,0 +1,149 @@ +require 'yaml' + +module Docs + # Reads the documents of the MDN content repository + # (https://github.com/mdn/content). Its pages are markdown files with a YAML + # front matter, sprinkled with the KumaScript macro calls that MDN expands + # when it builds its site. + module MdnContent + # The characters KumaScript strips when it turns the text of a heading into + # an id. See kumascript/src/api/util.ts in https://github.com/mdn/yari. + SECTION_ID_DISALLOWED = /["#$%&+,\/:;=?@\[\]^`{|}~')(\\]/ + + # Turns the text of a heading into the id MDN gives it, which is what the + # links to its section are pointing at. + def self.slugify(text) + text.strip.gsub(SECTION_ID_DISALLOWED, '').gsub(/\s+/, '_').gsub(/\A_+|_+\z/, '').downcase + end + + Page = Struct.new(:slug, :path, :front_matter) do + def title + front_matter['title'] + end + + def short_title + front_matter['short-title'] || title + end + + def page_type + front_matter['page-type'] + end + + def status + Array(front_matter['status']) + end + + def browser_compat + Array(front_matter['browser-compat']) + end + + def spec_urls + Array(front_matter['spec-urls']) + end + + def deprecated? + status.include?('deprecated') + end + + def non_standard? + status.include?('non-standard') + end + + def experimental? + status.include?('experimental') + end + end + + # The pages of one section of the content repository, indexed by the slug + # they hang off, e.g. "Global_Objects/Array/map" for the page whose front + # matter reads "slug: Web/JavaScript/Reference/Global_Objects/Array/map". + # The section's own page is indexed under the empty slug. + class Pages + include Enumerable + + FRONT_MATTER_DELIMITER = '---' + INDEX = 'index.md' + + # The part of the slug of every page that the directory stands for. + attr_reader :prefix + + def initialize(directory, prefix) + @directory = directory + @prefix = prefix + end + + def each(&block) + index.each_value(&block) + end + + def [](slug) + index[slug.to_s.downcase] + end + + def include?(slug) + index.key?(slug.to_s.downcase) + end + + def root + index[''] + end + + # The pages one level below a slug, in alphabetical order. + def children(slug) + below = slug.to_s.downcase + below += '/' unless below.empty? + index.select { |key, _| key.start_with?(below) && key.length > below.length && !key[below.length..].include?('/') }.values + end + + # Reads the markdown of a page, without its front matter. + def body(page) + File.read(File.join(@directory, page.path)).sub(/\A#{FRONT_MATTER_DELIMITER}\n.*?\n#{FRONT_MATTER_DELIMITER}\n/m, '') + end + + private + + def index + @index ||= build + end + + def build + pages = {} + + Dir.glob(File.join(@directory, '**', INDEX)).sort.each do |file| + front_matter = read_front_matter(file) + slug = front_matter['slug'] + next unless slug + + path = file.sub(%r{\A#{Regexp.escape(@directory)}/?}, '') + slug = slug.sub(%r{\A#{Regexp.escape(@prefix)}/?}, '') + pages[slug.downcase] = Page.new(slug, path, front_matter) + end + + pages + end + + # Only the front matter is read here: the pages are parsed in parallel + # later on, and reading all of them upfront would undo that. + def read_front_matter(file) + lines = [] + first = true + + File.foreach(file) do |line| + if first + return {} unless line.start_with?(FRONT_MATTER_DELIMITER) + first = false + elsif line.start_with?(FRONT_MATTER_DELIMITER) + break + else + lines << line + end + end + + # The selectors of the CSS reference start with a colon, which YAML + # reads as a symbol; they're written back the way they came. + front_matter = YAML.safe_load(lines.join, permitted_classes: [Symbol]) || {} + front_matter.transform_values { |value| value.is_a?(Symbol) ? ":#{value}" : value } + end + end + end +end diff --git a/lib/docs/scrapers/mdn/css.rb b/lib/docs/scrapers/mdn/css.rb index bcf34dceb4..33cbd2f21e 100644 --- a/lib/docs/scrapers/mdn/css.rb +++ b/lib/docs/scrapers/mdn/css.rb @@ -1,40 +1,21 @@ module Docs - class Css < Mdn - # release = '2025-09-15' + class Css < MdnGit + # release = '2026-09-14' self.name = 'CSS' - self.base_url = 'https://developer.mozilla.org/en-US/docs/Web/CSS' - self.root_path = '/Reference' + self.base_url = 'https://developer.mozilla.org/en-US/docs/Web/CSS/Reference' + self.content_path = 'web/css/reference' + self.slug_prefix = 'Web/CSS/Reference' self.links = { home: 'https://developer.mozilla.org/en-US/docs/Web/CSS', code: 'https://github.com/mdn/content/tree/main/files/en-us/web/css' } - html_filters.push 'css/clean_html', 'css/entries' + # The formal syntax and the table of characteristics of a property are + # MDN's, not its authors'. + self.data_packages = data_packages.merge('mdn-data' => 'mdn-data') - options[:root_title] = 'CSS' - - options[:skip] = %w(/CSS3 /Media/Visual /paged_media /Media/TV /Media/Tactile) - options[:skip] += %w(/mq-boolean /single-transition-timing-function) # bug - options[:skip_patterns] = [/Extensions/, /Tools/, /@media\/-webkit/, /webkit-mask/, /-moz-system-metric/] + html_filters.push 'css/entries' - options[:replace_paths] = { - '/%3Cbasic-shape%3E' => '/basic-shape', - '/fallback' => '/@counter-style/fallback', - '/range' => '/@counter-style/range', - '/symbols' => '/@counter-style/symbols', - '/system' => '/@counter-style/system', - '/var' => '/var()', - '/element' => '/element()', - '/Flexbox' => '/CSS_Flexible_Box_Layout/Using_CSS_flexible_boxes', - '/flexbox' => '/CSS_Flexible_Box_Layout/Using_CSS_flexible_boxes', - '/currentColor' => '/color_value' - } - - options[:fix_urls] = ->(url) do - url.sub! %r{https://developer\.mozilla\.org/en\-US/docs/CSS/([\w\-@:])}, "#{Css.base_url}/\\1" - url.sub! '%3A', ':' - url.sub! '%40', '@' - url - end + options[:root_title] = 'CSS' end end diff --git a/lib/docs/scrapers/mdn/dom.rb b/lib/docs/scrapers/mdn/dom.rb index e246683a75..42d1927f26 100644 --- a/lib/docs/scrapers/mdn/dom.rb +++ b/lib/docs/scrapers/mdn/dom.rb @@ -1,17 +1,18 @@ module Docs - class Dom < Mdn - # release = '2025-09-15' + class Dom < MdnGit + # release = '2026-09-14' self.name = 'Web APIs' self.slug = 'dom' self.base_url = 'https://developer.mozilla.org/en-US/docs/Web/API' + self.content_path = 'web/api' + self.slug_prefix = 'Web/API' self.links = { home: 'https://developer.mozilla.org/en-US/docs/Web/API', code: 'https://github.com/mdn/content/tree/main/files/en-us/web/api' } - html_filters.push 'dom/clean_html', 'dom/entries' + html_filters.push 'dom/entries' options[:root_title] = 'Web APIs' - end end diff --git a/lib/docs/scrapers/mdn/javascript.rb b/lib/docs/scrapers/mdn/javascript.rb index c99c656176..1af30f50e4 100644 --- a/lib/docs/scrapers/mdn/javascript.rb +++ b/lib/docs/scrapers/mdn/javascript.rb @@ -1,30 +1,22 @@ module Docs - class Javascript < Mdn - prepend FixInternalUrlsBehavior - prepend FixRedirectionsBehavior - - # release = '2026-08-13' + class Javascript < MdnGit + # release = '2026-09-14' self.name = 'JavaScript' self.base_url = 'https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference' + self.content_path = 'web/javascript/reference' + self.slug_prefix = 'Web/JavaScript/Reference' self.links = { home: 'https://developer.mozilla.org/en-US/docs/Web/JavaScript', code: 'https://github.com/mdn/content/tree/main/files/en-us/web/javascript' } - html_filters.push 'javascript/clean_html', 'javascript/entries' + html_filters.push 'javascript/entries' options[:root_title] = 'JavaScript' - # Duplicates - options[:skip] = %w( - /Global_Objects - /Operators - /Statements) - - options[:skip_patterns] = [ - /contributors.txt/, - /Deprecated_and_obsolete_features/ - ] + # Pages that repeat what their subpages or the reference index already say. + options[:skip] = %w(/Global_Objects /Operators /Statements) + options[:skip_patterns] = [/Deprecated_and_obsolete_features/] options[:fix_urls] = ->(url) do url.sub! '%2A', '*' diff --git a/lib/docs/scrapers/mdn/mdn_git.rb b/lib/docs/scrapers/mdn/mdn_git.rb new file mode 100644 index 0000000000..9fe6e0a24c --- /dev/null +++ b/lib/docs/scrapers/mdn/mdn_git.rb @@ -0,0 +1,154 @@ +require 'docs/mdn_content/data' +require 'docs/mdn_content/generator' +require 'docs/mdn_content/macros' +require 'docs/mdn_content/markdown' +require 'docs/mdn_content/pages' + +module Docs + # The MDN documentations built from the markdown of the content repository + # (https://github.com/mdn/content) rather than from the pages MDN serves, + # which are those same documents with their sidebars, compatibility tables + # and specifications rendered into them. Everything the scrapers need is one + # download away, so a documentation is built in the time the crawler used to + # spend waiting on developer.mozilla.org. + # + # The documentations that haven't moved over yet inherit Mdn, which + # crawls developer.mozilla.org. + class MdnGit < FileScraper + self.abstract = true + self.type = 'mdn' + self.links = { + home: 'https://developer.mozilla.org', + code: 'https://github.com/mdn/content' + } + + class << self + # The directory of the content repository holding the pages, relative to + # its files/en-us, and the slug that directory stands for. + attr_accessor :content_path, :slug_prefix + + # The npm packages MDN builds its compatibility tables, its list of + # specifications and its Baseline indicators out of. A documentation can + # add the ones it needs; see Css. + attr_accessor :data_packages + + def inherited(subclass) + super + subclass.content_path = content_path + subclass.slug_prefix = slug_prefix + subclass.data_packages = data_packages.dup + end + end + + self.data_packages = { + 'browser-compat-data' => '@mdn/browser-compat-data', + 'web-features' => 'web-features', + 'web-specs' => 'web-specs' + } + + html_filters.insert_before 'normalize_urls', 'mdn_git/macros', 'mdn_git/clean_html' + + options[:trailing_slash] = false + + # The links MDN's authors write aren't escaped, and the id of a heading is + # its text: a section called "Guideline 1.1 — providing text alternatives" + # is linked to by a fragment holding that em dash. + options[:fix_urls_before_parse] = ->(url) do + url.gsub(/[^\x21-\x7E]/) { |char| char.bytes.map { |byte| format('%%%02X', byte) }.join } + end + + options[:attribution] = <<-HTML + © 2005–2025 MDN contributors.
    + Licensed under the Creative Commons Attribution-ShareAlike License v2.5 or later. + HTML + + def get_latest_version(opts) + get_latest_github_commit_date('mdn', 'content', opts) + end + + # Every page is known upfront, so there's nothing to crawl: the scraper is + # handed the whole documentation and never has to follow a link to find a + # page. + def initial_paths + @initial_paths ||= pages.map(&:slug).reject { |slug| slug.empty? || skip?(slug) }.sort + end + + def build_pages(&block) + # Reading the pages and the data files here rather than in the forked + # workers means doing it once instead of once per job. + initial_paths + options + super + end + + private + + def pages + @pages ||= begin + assert_source_directory_exists + MdnContent::Pages.new source_directory, self.class.slug_prefix + end + end + + def generator + @generator ||= MdnContent::Generator.new(MdnContent::Data.new(data_directory)) + end + + def data_directory + File.join source_directory, '_data' + end + + def skip?(slug) + slug = "/#{slug}" + Array(self.class.options[:skip]).any? { |value| slug.casecmp(value) == 0 } || + Array(self.class.options[:skip_patterns]).any? { |pattern| slug.match?(pattern) } + end + + def page_for(url) + slug = url.to_s.remove(self.class.base_url).delete_prefix('/') + pages[slug] || raise("no page for #{url}") + end + + def url_to_path(url) + page_for(url).path + end + + def parse(response) + page = page_for response.url + html = generator.header(page) + MdnContent::Markdown.render(pages.body(page)) + [Parser.new(html).html, page.title] + end + + def additional_options + { pages: pages, generator: generator } + end + + def pipeline_context(response) + super.merge page: page_for(response.url) + end + + def download_source + download_and_extract 'https://github.com/mdn/content/archive/refs/heads/main.tar.gz', + "content-main/files/en-us/#{self.class.content_path}" + + self.class.data_packages.each do |directory, package| + download_and_extract npm_tarball(package), 'package', destination: File.join(data_directory, directory) + end + + instrument 'info.doc', msg: 'Reducing the data files to what the documentation needs...' + MdnContent::Data.prepare data_directory, compat_namespaces + end + + # The sections of browser-compat-data the pages query, e.g. "javascript" + # and "api". Everything else is dropped, which is most of it. + def compat_namespaces + pages.flat_map { |page| page.browser_compat.map { |query| query.split('.').first } }.uniq + end + + def npm_tarball(package) + response = Request.run "https://registry.npmjs.org/#{package}/latest" + raise SetupError, %(Failed to look up "#{package}" on npm) unless response.success? + JSON.parse(response.body).dig('dist', 'tarball') + end + end +end