Build the JavaScript, CSS and Web APIs documentations from the MDN content repository

Gemfile

@@ -35,6 +35,8 @@ group :development do
 end
 
 group :docs do
+  gem 'kramdown'
+  gem 'kramdown-parser-gfm'
   gem 'redcarpet'
 end
 

Gemfile.lock

@@ -134,6 +134,10 @@ GEM
       rdoc (>= 4.0.0)
       reline (>= 0.4.2)
     json (2.19.4)
+    kramdown (2.5.2)
+      rexml (>= 3.4.4)
+    kramdown-parser-gfm (1.1.0)
+      kramdown (~> 2.0)
     logger (1.7.0)
     loofah (2.25.1)
       crass (~> 1.0.2)
@@ -311,6 +315,8 @@ DEPENDENCIES
   html-pipeline (~> 2.14)
   image_optim
   image_optim_pack
+  kramdown
+  kramdown-parser-gfm
   minitest
   newrelic_rpm
   nokogiri
@@ -390,6 +396,8 @@ CHECKSUMS
   io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc
   irb (1.18.0) sha256=de9454a0703a54704b9811a5ef31a60c86949fbf4013fcf244fabc7c775248e3
   json (2.19.4) sha256=670a7d333fb3b18ca5b29cb255eb7bef099e40d88c02c80bd42a3f30fe5239ac
+  kramdown (2.5.2) sha256=1ba542204c66b6f9111ff00dcc26075b95b220b07f2905d8261740c82f7f02fa
+  kramdown-parser-gfm (1.1.0) sha256=fb39745516427d2988543bf01fc4cf0ab1149476382393e0e9c48592f6581729
   logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203
   loofah (2.25.1) sha256=d436c73dbd0c1147b16c4a41db097942d217303e1f7728704b37e4df9f6d2e04
   method_source (1.1.0) sha256=181301c9c45b731b4769bc81e8860e72f9161ad7d66dd99103c9ab84f560f5c5

lib/docs/core/scrapers/file_scraper.rb

@@ -199,13 +199,16 @@ def download_source
     end
 
     # Downloads an archive and moves it into #source_directory. Pass the
-    # subdirectory holding the documents when they aren't at the archive's root.
-    def download_and_extract(url, subdirectory = nil)
+    # subdirectory holding the documents when they aren't at the archive's root,
+    # and a destination to unpack somewhere else than #source_directory. Note
+    # that the destination is replaced, so the one holding the documents has to
+    # be unpacked before those nested inside it.
+    def download_and_extract(url, subdirectory = nil, destination: source_directory)
       instrument 'info.doc', msg: %(Downloading #{url}...)
       archive = Archive.download(url)
 
-      instrument 'info.doc', msg: %(Extracting the documentation files to "#{source_directory}"...)
-      Archive.unpack(archive, source_directory, directory: subdirectory)
+      instrument 'info.doc', msg: %(Extracting the documentation files to "#{destination}"...)
+      Archive.unpack(archive, destination, directory: subdirectory)
     ensure
       FileUtils.rm_f(archive) if archive
     end

lib/docs/filters/css/entries.rb

@@ -1,146 +1,78 @@
 module Docs
   class Css
     class EntriesFilter < Docs::EntriesFilter
-      TYPE_BY_PATH = {
-        'CSS_Animations' => 'Animations & Transitions',
-        'CSS_Background_and_Borders' => 'Backgrounds & Borders',
-        'CSS_Columns' => 'Multi-column Layout',
-        'CSS_Flexible_Box_Layout' => 'Flexible Box Layout',
-        'CSS_Fonts' => 'Fonts',
-        'CSS_Grid_Layout' => 'Grid Layout',
-        'CSS_Images' => 'Images',
-        'CSS_Lists_and_Counters' => 'Lists',
-        'CSS_Transforms' => 'Transforms',
-        'Media_Queries' => 'Media Queries',
-        'filter-function' => 'Filter Effects',
-        'transform-function' => 'Transforms',
-        '@media' => 'Media Queries',
-        'overscroll' => 'Overscroll',
-        'text-size-adjust' => 'Miscellaneous',
-        'resolved_value' => 'Miscellaneous',
-        'touch-action' => 'Miscellaneous',
-        'will-change' => 'Miscellaneous'
+      # What the specifications call themselves and what the documentation
+      # files them under.
+      TYPE_BY_SPEC = {
+        'Compositing' => 'Compositing & Blending',
+        'Custom Properties for Cascading Variables' => 'Variables',
+        'Grid Layout' => 'Grid Layout',
+        'Image Values & Replaced Content' => 'Image Values',
+        'Scroll Snap' => 'Scroll Snap'
       }
 
-      DATA_TYPE_SLUGS = %w(angle basic-shape color_value counter frequency
-        gradient image integer length number percentage position_value ratio
-        resolution shape string time timing-function uri user-ident)
+      # The kind of page to fall back on when no specification names it.
+      TYPE_BY_PAGE_TYPE = {
+        'css-at-rule' => 'At-rules',
+        'css-at-rule-descriptor' => 'At-rules',
+        'css-combinator' => 'Selectors',
+        'css-function' => 'Functions',
+        'css-keyword' => 'Keywords',
+        'css-media-feature' => 'Media Queries',
+        'css-property' => 'Properties',
+        'css-pseudo-class' => 'Selectors',
+        'css-pseudo-element' => 'Pseudo-Elements',
+        'css-selector' => 'Selectors',
+        'css-shorthand-property' => 'Properties',
+        'css-type' => 'Data Types'
+      }
+
+      VENDOR_PREFIXES = %w(-webkit- -moz- -ms- -o-)
 
-      FUNCTION_SLUGS = %w(attr calc cross-fade cubic-bezier cycle element
-        linear-gradient radial-gradient repeating-linear-gradient
-        repeating-radial-gradient var)
+      # The descriptors and the media features of an at-rule go by their own
+      # name in the content repository; the reference lists them under the
+      # at-rule they belong to.
+      NESTED = %w(css-at-rule-descriptor css-media-feature)
 
       def get_name
-        if DATA_TYPE_SLUGS.include?(slug)
-          "<#{super.remove ' value'}>"
-        elsif FUNCTION_SLUGS.include?(slug)
-          "#{super}()"
-        elsif slug =~ /\A[a-z]+_/i
-          slug.to_s.gsub('_', ' ').gsub('/', ': ')
-        elsif slug.start_with?('transform-function') || slug.start_with?('filter-function')
-          slug.split('/').last + '()'
-        else
-          super
-        end
+        name = page.front_matter['short-title'] || default_name
+        at_rule ? "#{at_rule}.#{name}" : name
       end
 
-      def get_type
-        if slug.include?('-webkit') || slug.include?('-moz') || slug.include?('-ms')
-          'Extensions'
-        elsif type = TYPE_BY_PATH[slug.split('/').first]
-          type
-        elsif type = get_spec
-          type.remove! 'CSS '
-          type.remove! ' Module'
-          type.remove! %r{ Level \d\z}
-          type.remove! %r{\(.*\)}
-          type.remove! %r{  \d\z}
-          type.sub! 'and', '&'
-          type.strip!
-          type = 'Grid Layout' if type.include?('Grid Layout')
-          type = 'Scroll Snap' if type.include?('Scroll Snap')
-          type = 'Compositing & Blending' if type.include?('Compositing')
-          type = 'Animations & Transitions' if type.in?(%w(Animations Transitions))
-          type = 'Image Values' if type == 'Image Values & Replaced Content'
-          type = 'Variables' if type == 'Custom Properties for Cascading Variables'
-          type.prepend 'Miscellaneous ' if type =~ /\ALevel \d\z/
-          type
-        elsif name.start_with?('::')
-          'Pseudo-Elements'
-        elsif name.start_with?(':')
-          'Selectors'
-        elsif name.start_with?('display-')
-          'Display'
-        else
-          'Miscellaneous'
-        end
+      def default_name
+        page.page_type.to_s.start_with?('css-') ? slug.split('/').last : page.title
       end
 
-      STATUSES = {
-        'spec-Living'   => 0,
-        'spec-REC'      => 1,
-        'spec-CR'       => 2,
-        'spec-PR'       => 3,
-        'spec-LC'       => 4,
-        'spec-WD'       => 5,
-        'spec-ED'       => 6,
-        'spec-Obsolete' => 7
-      }
-
-      PRIORITY_STATUSES = %w(spec-REC spec-CR)
-      PRIORITY_SPECS = ['CSS Basic Box Model', 'CSS Lists and Counters', 'CSS Paged Media']
-
-      def get_spec
-        return unless table = at_css('#Specifications + table') || css('.standard-table').last
-
-        specs = table.css('tbody tr').to_a
-        # [link, span]
-        specs.map!     { |node| [node.at_css('> td:nth-child(1) > a'), node.at_css('> td:nth-child(2) > span')] }
-        # ignore non-CSS specs
-        specs.select!  { |pair| pair.first && pair.first['href'] =~ /css|fxtf|fullscreen|svg/i && !pair.first['href'].include?('compat.spec') }
-        # ignore specs with no status
-        specs.select!  { |pair| pair.second }
-        # ["Spec", "spec-REC"]
-        specs.map!     { |pair| [pair.first.child.content, pair.second['class']] }
-        # sort by status
-        specs.sort_by! { |pair| [STATUSES[pair.second], pair.first] }
-
-        spec = specs.find { |pair| PRIORITY_SPECS.any? { |s| pair.first.start_with?(s) } && name != 'display' }
-        spec ||= specs.find { |pair| !pair.first.start_with?('CSS Level') && pair.second.in?(PRIORITY_STATUSES) }
-        spec ||= specs.find { |pair| pair.second == 'spec-WD' } if specs.count { |pair| pair.second == 'spec-WD' } == 1
-        spec ||= specs.first
+      def at_rule
+        return unless NESTED.include?(page.page_type)
+        context[:pages][slug.split('/')[0..-2].join('/')]&.short_title
+      end
 
-        spec.try(:first)
+      def get_type
+        return 'Extensions' if VENDOR_PREFIXES.any? { |prefix| name.include?(prefix) }
+        specification || TYPE_BY_PAGE_TYPE[page.page_type] || 'Miscellaneous'
       end
 
-      ADDITIONAL_ENTRIES = {
-        'shape' => [
-          %w(rect() Syntax) ],
-        'timing-function' => [
-          %w(cubic-bezier()),
-          %w(steps()),
-          %w(linear linear),
-          %w(ease ease),
-          %w(ease-in ease-in),
-          %w(ease-in-out ease-in-out),
-          %w(ease-out ease-out),
-          %w(step-start step-start),
-          %w(step-end step-end) ],
-        'color_value' => [
-          %w(transparent transparent),
-          %w(currentColor currentColor),
-          %w(rgb() rgba()),
-          %w(hsl() hsla()),
-          %w(rgba() rgba()),
-          %w(hsla() hsla()) ]}
+      # The specification a page documents, cut down to the part of CSS it
+      # covers: "CSS Grid Layout Module Level 2" is the Grid Layout pages.
+      def specification
+        title = context[:generator].specification_titles(page.browser_compat, page.spec_urls).first
+        return if title.blank?
+
+        title = title.dup
+        title.remove! 'CSS '
+        title.remove! ' Module'
+        title.remove! %r{ Level \d+\z}
+        title.remove! %r{\(.*\)}
+        title.sub! ' and ', ' & '
+        title.squish!
 
-      def additional_entries
-        ADDITIONAL_ENTRIES[slug] || []
+        _, type = TYPE_BY_SPEC.find { |spec, _| title.include?(spec) }
+        type || (title.match?(/\ALevel \d+\z/) ? nil : title.presence)
       end
 
-      def include_default_entry?
-        return true unless warning = at_css('.warning').try(:content)
-        !warning.include?('CSS Flexible Box') && !warning.include?('replaced in newer drafts')
+      def page
+        context[:page]
       end
     end
   end