`` or its ````."""
+ for el in (pre, *pre.iter("code")):
+ if m := _LANG_CLASS.search(el.get("class", "")):
+ return m.group(1).lower()
+ return ""
+
+
+def _callout(el: HtmlElement) -> tuple[str, str, str, str] | None:
+ """(kind, title, body markdown, body text) of a ``Tbody`` box."""
+ m = _CALLOUT_CLASS.search(el.get("class", ""))
+ if el.tag != "div" or not m or not _inline_only(el):
+ return None
+ title = ""
+ if len(el) and el[0].tag in ("b", "strong") and not (el.text or "").strip():
+ title = _collapse(el[0].text_content())
+ md = _collapse(_inline_md(el))
+ text = _inline_text(el)
+ if title:
+ md = md.removeprefix(f"**{title}**").strip()
+ text = text.removeprefix(title).strip()
+ kind = _CALLOUT_KINDS[m.group(1).lower()]
+ return (kind, title, md, text) if md or title else None
+
+
+def _banner(el: HtmlElement) -> tuple[str, str] | None:
+ """(title, subtitle) of a stage/chapter banner: ``TS``."""
+ if el.tag != "div" or not _BANNER_CLASS.search(el.get("class", "")):
+ return None
+ kids = [c for c in el if isinstance(c.tag, str)]
+ if not kids or kids[0].tag not in ("b", "strong") or any(c.tag != "span" for c in kids[1:]):
+ return None
+ title = _collapse(kids[0].text_content())
+ return (title, _collapse(" ".join(c.text_content() for c in kids[1:]))) if title else None
+
+
+def _mark_blocks(root: HtmlElement) -> None:
+ """Flag every element that has a block-level descendant, in one pass over the tree."""
+ for el in reversed(list(root.iter())): # reverse document order: children before parents
+ parent = el.getparent()
+ if parent is None or not isinstance(el.tag, str):
+ continue
+ if el.tag.lower() in _BLOCK_TAGS or el.get(_BLOCK_ATTR) is not None:
+ parent.set(_BLOCK_ATTR, "1")
+
+
+def _inline_only(el: HtmlElement) -> bool:
+ """Whether ``el`` holds no block-level element (the tree went through ``_mark_blocks``)."""
+ return el.get(_BLOCK_ATTR) is None
+
+
+def _paragraph(sec: HeadingSectioner, el: HtmlElement, *, bold: bool = False) -> None:
+ md, text = _collapse(_inline_md(el)), _inline_text(el)
+ if md and text:
+ sec.add_paragraph(f"**{text}**" if bold else md, text)
+
+
+def _stray(sec: HeadingSectioner, text: str | None) -> None:
+ """Text sitting directly between a container's block children."""
+ if text and text.strip():
+ sec.add_paragraph(_collapse(text), _collapse(text))
+
+
+def _walk(el: HtmlElement, sec: HeadingSectioner, attach: dict[str, list[ScriptBlock]]) -> None:
+ _stray(sec, el.text)
+ for child in el:
+ _walk_child(child, sec, attach)
+ _stray(sec, child.tail)
+
+
+def _walk_child(
+ child: HtmlElement, sec: HeadingSectioner, attach: dict[str, list[ScriptBlock]]
+) -> None:
+ if not isinstance(child.tag, str):
+ return
+ tag = child.tag.lower()
+ if tag in _SKIP:
+ _attach(child, sec, attach) # a JSON island's own data is placed where it sits
+ return
+ if tag in _HEADINGS:
+ sec.add_heading(_HEADINGS[tag], _heading_text(child))
+ elif tag == "p":
+ _paragraph(sec, child, bold=bool(_EMPHASIS_CLASS.search(child.get("class", ""))))
+ elif tag in ("ul", "ol"):
+ _list_items(child, sec, 0, tag == "ol", attach)
+ elif tag == "pre":
+ sec.add_code(child.text_content(), _code_lang(child))
+ elif tag == "blockquote":
+ sec.add_quote(_inline_text(child), _collapse(_inline_md(child)))
+ elif tag == "table":
+ md, cells = _table_md(child)
+ sec.add_table(md, cells)
+ elif (callout := _callout(child)) is not None:
+ sec.add_callout(*callout)
+ elif (banner := _banner(child)) is not None:
+ sec.add_heading(2, banner[0])
+ if banner[1]:
+ sec.add_paragraph(banner[1], banner[1])
+ elif tag == "summary":
+ _paragraph(sec, child, bold=True)
+ elif (tag in _CONTAINERS or len(child)) and tag != "dl" and _inline_only(child):
+ _paragraph(sec, child)
+ elif tag in _CONTAINERS or len(child):
+ _walk(child, sec, attach)
+ _attach(child, sec, attach)
+ return
+ else:
+ text = _collapse(child.text_content())
+ if text:
+ sec.add_paragraph(text, text)
+ _attach(child, sec, attach)
+ for island in child.iterdescendants("script"): # flattened above, so never walked
+ _attach(island, sec, attach)
+
+
+def _attach(el: HtmlElement, sec: HeadingSectioner, attach: dict[str, list[ScriptBlock]]) -> None:
+ """Emit the script blocks named by ``el``'s ``id`` or a ``data-*`` attribute."""
+ for name, value in el.attrib.items(): # an empty value (data-v-1a2b="") names nothing
+ if value and (name == "id" or name.startswith("data-")) and value in attach:
+ blocks = attach.pop(value)
+ size = sum(len(b.text) + sum(map(len, chain(*b.rows))) for b in blocks)
+ # A widget's data reads as part of its section; a whole dataset gets sections.
+ level = min(max(sec.depth + 1, 2), 6) if size > _INLINE_DATA_CHARS else 0
+ add_blocks(sec, blocks, level=level)
+
+def walk_html(
+ root: HtmlElement, sec: HeadingSectioner, attach: dict[str, list[ScriptBlock]] | None = None
+) -> None:
+ """Feed the reading-order content of ``root`` into ``sec``.
-def walk_html(root: HtmlElement, sec: HeadingSectioner) -> None:
- """Feed the reading-order content of ``root`` into ``sec``."""
+ ``attach`` maps an attribute value to script blocks emitted after the element
+ carrying it; entries not claimed by any element are left in the dict.
+ """
+ _mark_blocks(root)
body = root.body if root.tag == "html" and root.body is not None else root
- _walk(body, sec)
+ _walk(body, sec, {} if attach is None else attach)
diff --git a/fnd/extract/_js_data.py b/fnd/extract/_js_data.py
new file mode 100644
index 00000000..f1b84fa6
--- /dev/null
+++ b/fnd/extract/_js_data.py
@@ -0,0 +1,327 @@
+"""Tolerant reader for the data literals in inline scripts (nothing is executed).
+
+``parse_assignments`` finds ``var name = {...}``, ``name = {...}`` and
+``X.name = [...]`` literals and returns them as Python values. Anything that is not plain data (calls,
+functions, member access) becomes ``OPAQUE`` and the parse carries on, so a
+page's computed cells never hide the literal data around them. Identifiers
+that name an earlier ``var``/``let``/``const`` literal resolve to its value.
+"""
+
+from __future__ import annotations
+
+import re
+from bisect import bisect_left
+from typing import Final, NamedTuple
+
+OPAQUE: Final = object()
+
+_MAX_DEPTH = 40
+_TOKEN = re.compile(
+ r"""
+ (?P\s+|//[^\n]*|/\*(?:.*?\*/|.*))
+ | (?P"(?:[^"\\\n]|\\.)*(?:"|(?=\n)|\Z)|'(?:[^'\\\n]|\\.)*(?:'|(?=\n)|\Z))
+ | (?P`(?:[^`\\]|\\.)*(?:`|\Z))
+ | (?P-?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?(?![\w$]))
+ | (?P[\w$]+)
+ | (?P[{}\[\](),:;])
+ | (?P.)
+ """,
+ re.S | re.X,
+)
+_ASSIGN = re.compile(
+ r"(?:(? str:
+ def sub(m: re.Match[str]) -> str:
+ digits = m.group(1) or m.group(2)
+ return chr(int(digits, 16)) if digits else _ESCAPES.get(m.group(3), m.group(3))
+
+ text = _ESCAPE.sub(sub, raw)
+ if _SURROGATE.search(
+ text
+ ): # a pair of \uXXXX escapes is one character; a lone half is not text
+ text = text.encode("utf-16", "surrogatepass").decode("utf-16", "replace")
+ return text
+
+
+def _tokens(src: str) -> list[_Tok]:
+ out: list[_Tok] = []
+ for m in _TOKEN.finditer(src):
+ kind, text = m.lastgroup or "other", m.group()
+ if kind in ("str", "tpl") and not _closed(text):
+ kind = "other" # an unterminated literal swallowed its line; keep it out of the data
+ if kind != "skip":
+ out.append(_Tok(kind, text, m.start(), m.end()))
+ return out
+
+
+def _closed(literal: str) -> bool:
+ inner = literal[1:-1]
+ return (
+ len(literal) > 1
+ and literal[-1] == literal[0]
+ and (len(inner) - len(inner.rstrip("\\"))) % 2 == 0
+ )
+
+
+class _Expansion:
+ """Size of the values a script refers to by name, so a shared reference cannot multiply."""
+
+ def __init__(self) -> None:
+ self.left = _MAX_EXPANSION
+ self._weights: dict[int, int] = {}
+
+ def weight(self, value: object) -> int:
+ if isinstance(value, str):
+ return 1 + len(value) // 16
+ if not isinstance(value, (list, dict, Call)):
+ return 1
+ known = self._weights.get(id(value))
+ if known is None:
+ parts = (
+ value.values()
+ if isinstance(value, dict)
+ else value.args
+ if isinstance(value, Call)
+ else value
+ )
+ known = min(1 + sum(self.weight(p) for p in parts), _MAX_EXPANSION + 1)
+ self._weights[id(value)] = known
+ return known
+
+ def take(self, value: object) -> bool:
+ """Charge one more use of ``value``; False once the script has expanded too far."""
+ cost = self.weight(value)
+ if cost > self.left:
+ return False
+ self.left -= cost
+ return True
+
+
+class _Parser:
+ def __init__(
+ self, toks: list[_Tok], env: dict[str, object], expansion: _Expansion | None = None
+ ) -> None:
+ self.toks = toks
+ self.env = env
+ self.expansion = expansion or _Expansion()
+ self.i = 0
+
+ def _peek(self) -> _Tok | None:
+ return self.toks[self.i] if self.i < len(self.toks) else None
+
+ def _at(self, text: str) -> bool:
+ t = self._peek()
+ return t is not None and t.kind in ("punct", "other") and t.text == text
+
+ def _skip_expr(self) -> object:
+ """Consume to the next depth-0 terminator; the expression is not data."""
+ depth = 0
+ while (t := self._peek()) is not None:
+ if t.kind == "punct" and t.text in "([{":
+ depth += 1
+ elif t.kind == "punct" and t.text in ")]}":
+ if depth == 0:
+ break
+ depth -= 1
+ elif depth == 0 and t.kind == "punct" and t.text in ",;":
+ break
+ self.i += 1
+ return OPAQUE
+
+ def value(self, depth: int = 0) -> object:
+ t = self._peek()
+ if t is None or depth > _MAX_DEPTH:
+ return self._skip_expr()
+ v: object
+ if t.kind == "str":
+ self.i += 1
+ v = unescape(t.text[1:-1])
+ elif t.kind == "tpl":
+ self.i += 1
+ v = OPAQUE if "${" in t.text else unescape(t.text[1:-1])
+ elif t.kind == "num":
+ self.i += 1
+ v = float(t.text) if any(c in t.text for c in ".eE") else int(t.text)
+ elif t.kind == "punct" and t.text == "[":
+ v = self._array(depth + 1)
+ elif t.kind == "punct" and t.text == "{":
+ v = self._object(depth + 1)
+ elif t.kind == "word":
+ self.i += 1
+ v = self._call(t.text, depth + 1) if self._at("(") else self._word(t.text)
+ else:
+ return self._skip_expr()
+ nxt = self._peek()
+ if nxt is None or (nxt.kind == "punct" and nxt.text in _END):
+ return v
+ if isinstance(v, str) and self._at("+"):
+ self.i += 1
+ rest = self.value(depth + 1)
+ return v + rest if isinstance(rest, str) else OPAQUE
+ return self._skip_expr()
+
+ def _word(self, text: str) -> object:
+ if text in ("true", "false"):
+ return text == "true"
+ if text in ("null", "undefined"):
+ return None
+ value = self.env.get(text, OPAQUE)
+ return value if self.expansion.take(value) else OPAQUE
+
+ def _items(self, closer: str, depth: int) -> list[object]:
+ self.i += 1
+ items: list[object] = []
+ while (t := self._peek()) is not None and not (t.kind == "punct" and t.text == closer):
+ before = self.i
+ items.append(self.value(depth))
+ if self._at(",") or self.i == before:
+ self.i += 1
+ self.i += 1
+ return items
+
+ def _array(self, depth: int) -> list[object]:
+ return self._items("]", depth)
+
+ def _call(self, name: str, depth: int) -> Call:
+ return Call(name, tuple(self._items(")", depth)))
+
+ def _object(self, depth: int) -> dict[str, object]:
+ self.i += 1
+ obj: dict[str, object] = {}
+ while (t := self._peek()) is not None and not (t.kind == "punct" and t.text == "}"):
+ before = self.i
+ key = unescape(t.text[1:-1]) if t.kind == "str" else t.text
+ self.i += 1
+ if t.kind in ("word", "str", "num") and self._at(":"):
+ self.i += 1
+ obj[key] = self.value(depth)
+ else:
+ self.i = before + 1
+ self._skip_expr()
+ if self._at(",") or self.i == before:
+ self.i += 1
+ self.i += 1
+ return obj
+
+
+def parse_literal(src: str) -> object:
+ """The value of a bare literal such as a ``')
+
+
+def test_shell_steps_get_a_bash_fence(tmp_path: Path) -> None:
+ md = _flow(tmp_path, _PAGE_SCRIPT)
+ assert "```bash\nls /data\n```" in md
+
+
+def test_a_view_caption_with_no_subtitle_is_not_left_dangling(tmp_path: Path) -> None:
+ script = """FLOWS = {}; FLOWS.demo = { steps: [ { say: "Show the records so far.",
+ views: [{ type: "rows", label: "women", items: [] }] } ] };"""
+ md = _flow(tmp_path, script)
+ assert "*women*" in md
+ assert "· *" not in md
+
+
+def test_chip_rows_render_as_inline_code(tmp_path: Path) -> None:
+ script = """FLOWS = {}; FLOWS.demo = { steps: [ { say: "Two records, one per line.",
+ views: [{ type: "rows", label: "nums", sub: "RDD",
+ items: [C("1", "p0"), C("2", "p1")] }] } ] };"""
+ md = _flow(tmp_path, script)
+ assert "*nums · RDD*" in md
+ assert "`1` `2`" in md
+
+
+def test_row_mappings_render_as_a_table_with_the_views_own_headings(tmp_path: Path) -> None:
+ script = """FLOWS = {}; FLOWS.demo = { steps: [ { say: "Each record is upper-cased.",
+ views: [{ type: "map", inLabel: "raw", mid: "step", outLabel: "clean",
+ rows: [{ a: C("ab"), b: C("AB"), why: "upper" }] }] } ] };"""
+ md = _flow(tmp_path, script)
+ assert "| raw | clean | step |" in md
+ assert "| `ab` | `AB` | upper |" in md
+
+
+def _view_md(tmp_path: Path, view: str) -> str:
+ script = f"""FLOWS = {{}}; FLOWS.demo = {{ steps: [ {{ say: "Look at what the step did.",
+ views: [{view}] }} ] }};"""
+ return _flow(tmp_path, script)
+
+
+def test_an_html_view_is_walked_as_a_document_not_dumped_as_code(tmp_path: Path) -> None:
+ html = (
+ 'x[2] int(x[2]) < 25 1 yes
"
+ )
+ md = _view_md(tmp_path, f'{{ type: "html", html: "{html}" }}')
+ assert "| x[2] | int(x[2]) < 25 |" in md
+ assert "| 1 | yes |" in md
+ assert " None:
+ html = 'Row(x[0])'
+ md = _view_md(tmp_path, f'{{ type: "html", html: "{html}" }}')
+ assert "Row( **x[0]** )" in md
+ assert "<" not in md.partition("**Step 1**")[2]
+
+
+def test_a_true_flag_in_a_row_table_is_not_an_empty_cell(tmp_path: Path) -> None:
+ md = _view_md(tmp_path, '{ type: "groups", groups: [{ key: C("k"), sealed: true }] }')
+ assert "| `k` | yes |" in md
+
+
+def test_a_takeaway_paragraph_is_emphasised(tmp_path: Path) -> None:
+ md = _md(
+ tmp_path, 'Read only from the path you name.
Plain text here.
'
+ )
+ assert "**Read only from the path you name.**" in md
+ assert "**Plain text here.**" not in md
+
+
+def test_a_pipeline_of_plan_steps_renders_as_one_arrow_line(tmp_path: Path) -> None:
+ md = _view_md(
+ tmp_path, '{ type: "plan", steps: [{ name: "textFile" }, { name: "map split" }] }'
+ )
+ assert "`textFile` → `map split`" in md
+
+
+def test_a_dropped_row_says_so(tmp_path: Path) -> None:
+ md = _view_md(
+ tmp_path,
+ '{ type: "map", inLabel: "in", outLabel: "out", rows: [{ a: C("x"), b: null, why: "no" }] }',
+ )
+ assert "| `x` | dropped | no |" in md
+
+
+def test_gate_results_name_the_rule_that_matched(tmp_path: Path) -> None:
+ md = _view_md(
+ tmp_path,
+ '{ type: "gates", col: "x[2]", rules: ["a < 1", "else"], items: [{ v: "3", hit: 1, out: "B" }] }',
+ )
+ assert "| x[2] | first true rule | out |" in md
+ assert "**col**" not in md
+ assert "**rules**" not in md
+ assert "| 3 | `else` | B |" in md
+
+
+def test_partition_lists_render_one_line_per_partition(tmp_path: Path) -> None:
+ script = """FLOWS = {}; FLOWS.demo = { steps: [ { say: "The list is cut into pieces.",
+ views: [{ type: "parts", label: "rdd", parts: [[C("1"), C("2")], [C("3")]] }] } ] };"""
+ md = _flow(tmp_path, script)
+ assert "- **Part 0** `1` `2`" in md
+ assert "- **Part 1** `3`" in md
+
+
+def test_minified_sibling_elements_keep_their_words_apart(tmp_path: Path) -> None:
+ html = (
+ ""
+ "- Apple
- A fruit
- Brick
- A block
"
+ "Price$10"
+ )
+ body = _body(tmp_path, html)
+ for words in ("Home About", "Apple\nA fruit", "Brick\nA block", "Price $10"):
+ assert words in body
+
+
+def test_words_split_by_styling_stay_whole(tmp_path: Path) -> None:
+ html = (
+ 'os.path
'
+ "$10.99 and THE
"
+ "42
"
+ )
+ body = _body(tmp_path, html)
+ assert "os.path" in body
+ assert "$10.99" in body
+ assert "THE" in body
+ assert "42" in body
+
+
+def test_markdown_and_index_text_space_a_sentence_boundary_alike(tmp_path: Path) -> None:
+ html = "Note.Important stuff
"
+ assert "Note. Important stuff" in _body(tmp_path, html)
+ assert "*Note.* **Important** stuff" in _md(tmp_path, html)
+
+
+def test_inline_markup_after_a_block_in_a_list_item_stays_one_paragraph(tmp_path: Path) -> None:
+ md = _md(
+ tmp_path,
+ '- Run
make
then check the log output
',
+ )
+ assert "then `check` the [log](/log) output" in md
+
+
+def test_a_list_item_made_of_blocks_keeps_its_bullet(tmp_path: Path) -> None:
+ md = _md(tmp_path, "- AdaEngineer
")
+ assert "- Ada" in md
+ assert "Engineer" in md
+
+
+def test_a_standalone_button_label_is_indexed(tmp_path: Path) -> None:
+ html = 'An answer.'
+ assert "What is it?" in _body(tmp_path, html)
+
+
+def test_table_cells_keep_a_bare_less_than_and_url_ampersands(tmp_path: Path) -> None:
+ script = (
+ "var DATA = { rows: [{ a: 'xDoc