diff --git a/docs/changelog.md b/docs/changelog.md index 913732761..b29385e2d 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -20,6 +20,7 @@ See the [Contributing Guide](contributing.md) for details. performance for repeated inline patterns (#1619). * Officially support Python 3.15 and drop support for Python 3.10 * Walk backtick runs in `BacktickInlineProcessor` without a regex (#1620). +* Complete rework of emphasis handling to imporove nested emphasis handling better (#1632). ### Fixed diff --git a/docs/extensions/api.md b/docs/extensions/api.md index 6952bcd19..4822bbc2b 100644 --- a/docs/extensions/api.md +++ b/docs/extensions/api.md @@ -349,7 +349,7 @@ Here are some convenience functions and other examples: | Class | Kind | Description | | -------------------------------------------------------------------------------------|-----------|---------------------------------------------------------------| -| [`AsteriskProcessor`][markdown.inlinepatterns.AsteriskProcessor] | built-in | Emphasis processor for handling strong and em matches inside asterisks | +| [`DelimiterProcessor`][markdown.inlinepatterns.DelimiterProcessor] | built-in | Emphasis processor for handling strong and em matches | | [`WikiLinksInlineProcessor`][markdown.extensions.wikilinks.WikiLinksInlineProcessor] | extension | Link `[[article names]]` to wiki given in metadata | | [`FootnoteInlineProcessor`][markdown.extensions.footnotes.FootnoteInlineProcessor] | extension | Replaces footnote in text with link to footnote div at bottom | diff --git a/markdown/core.py b/markdown/core.py index 370cb7ec5..a04510c03 100644 --- a/markdown/core.py +++ b/markdown/core.py @@ -28,7 +28,7 @@ from .preprocessors import build_preprocessors from .blockprocessors import build_block_parser from .treeprocessors import build_treeprocessors -from .inlinepatterns import build_inlinepatterns +from .inlinepatterns import build_inlinepatterns, DelimiterProcessor from .postprocessors import build_postprocessors from .extensions import Extension from .serializers import to_html_string, to_xhtml_string @@ -106,6 +106,7 @@ def __init__(self, **kwargs): """ + self.last_run: float = 0.0 self.tab_length: int = kwargs.get('tab_length', 4) self.ESCAPED_CHARS: list[str] = [ @@ -118,6 +119,7 @@ def __init__(self, **kwargs): self.registeredExtensions: list[Extension] = [] self.docType = "" # TODO: Maybe delete this. It does not appear to be used anymore. self.stripTopLevelTags: bool = True + self.delimiters: dict[str, DelimiterProcessor] = {} self.build_parser() @@ -270,6 +272,12 @@ def reset(self) -> Markdown: self.htmlStash.reset() self.references.clear() + for key in list(self.delimiters): + ext = self.delimiters[key] + ext.reset() + if ext not in self.inlinePatterns: + del self.delimiters[key] + for extension in self.registeredExtensions: if hasattr(extension, 'reset'): extension.reset() diff --git a/markdown/extensions/legacy_em.py b/markdown/extensions/legacy_em.py index 39efe9a73..e0edf05b4 100644 --- a/markdown/extensions/legacy_em.py +++ b/markdown/extensions/legacy_em.py @@ -14,29 +14,7 @@ from __future__ import annotations from . import Extension -from ..inlinepatterns import UnderscoreProcessor, EmStrongItem, EM_STRONG2_RE, STRONG_EM2_RE -import re - -# _emphasis_ -EMPHASIS_RE = r'(_)([^_]+)\1' - -# __strong__ -STRONG_RE = r'(_{2})(.+?)\1' - -# __strong_em___ -STRONG_EM_RE = r'(_)\1(?!\1)([^_]+?)\1(?!\1)(.+?)\1{3}' - - -class LegacyUnderscoreProcessor(UnderscoreProcessor): - """Emphasis processor for handling strong and em matches inside underscores.""" - - PATTERNS = [ - EmStrongItem(re.compile(EM_STRONG2_RE, re.DOTALL | re.UNICODE), 'double', 'strong,em'), - EmStrongItem(re.compile(STRONG_EM2_RE, re.DOTALL | re.UNICODE), 'double', 'em,strong'), - EmStrongItem(re.compile(STRONG_EM_RE, re.DOTALL | re.UNICODE), 'double2', 'strong,em'), - EmStrongItem(re.compile(STRONG_RE, re.DOTALL | re.UNICODE), 'single', 'strong'), - EmStrongItem(re.compile(EMPHASIS_RE, re.DOTALL | re.UNICODE), 'single', 'em') - ] +from ..inlinepatterns import DelimiterProcessor class LegacyEmExtension(Extension): @@ -45,13 +23,13 @@ class LegacyEmExtension(Extension): def extendMarkdown(self, md): """ Register the processor. - | Class Instance | Registry | Name | Priority | - | ------------------------------------------------------------- | ---------------------------------------------------------------- | ------ | :------: | - | [`LegacyUnderscoreProcessor`][markdown.extensions.legacy_em.LegacyUnderscoreProcessor] | [`inlinepatterns`][markdown.inlinepatterns.build_inlinepatterns] | `em_strong2` | `50` | + | Class Instance | Registry | Name | Priority | + | ------------------------------------------------------------------ | ---------------------------------------------------------------- | ------------ | :------: | + | [`DelimiterProcessor`][markdown.inlinepatterns.DelimiterProcessor] | [`inlinepatterns`][markdown.inlinepatterns.build_inlinepatterns] | `em_strong2` | `50` | """ - # flake8: noqa: E501 48-50 - md.inlinePatterns.register(LegacyUnderscoreProcessor(r'_'), 'em_strong2', 50) + # flake8: noqa: E501 27-29 + md.inlinePatterns.register(DelimiterProcessor(r'_', 'strong,em', md), 'em_strong2', 50) def makeExtension(**kwargs): # pragma: no cover diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index 0f3533b2e..cb6dac412 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -41,7 +41,8 @@ from __future__ import annotations from . import util -from typing import TYPE_CHECKING, Any, Collection, NamedTuple +from typing import TYPE_CHECKING, Any, Collection, NamedTuple, cast +from collections import deque import re import xml.etree.ElementTree as etree from html import entities @@ -89,9 +90,8 @@ def build_inlinepatterns(md: Markdown, **kwargs: Any) -> util.Registry[InlinePro inlinePatterns.register(SubstituteTagInlineProcessor(LINE_BREAK_RE, 'br'), 'linebreak', 100) inlinePatterns.register(HtmlInlineProcessor(HTML_RE, md), 'html', 90) inlinePatterns.register(HtmlInlineProcessor(ENTITY_RE, md), 'entity', 80) - inlinePatterns.register(SimpleTextInlineProcessor(NOT_STRONG_RE), 'not_strong', 70) - inlinePatterns.register(AsteriskProcessor(r'\*'), 'em_strong', 60) - inlinePatterns.register(UnderscoreProcessor(r'_'), 'em_strong2', 50) + inlinePatterns.register(DelimiterProcessor('*', 'strong,em', md), 'em_strong', 60) + inlinePatterns.register(DelimiterProcessor('_', 'strong,em', md, smart=True), 'em_strong2', 50) return inlinePatterns @@ -107,36 +107,6 @@ def build_inlinepatterns(md: Markdown, **kwargs: Any) -> util.Registry[InlinePro ESCAPE_RE = r'\\(.)' """ Match a backslash escaped character (`\\<` or `\\*`). """ -EMPHASIS_RE = r'(\*)([^\*]+)\1' -""" Match emphasis with an asterisk (`*emphasis*`). """ - -STRONG_RE = r'(\*{2})(.+?)\1' -""" Match strong with an asterisk (`**strong**`). """ - -SMART_STRONG_RE = r'(?)` or `[text](url "title")`). """ @@ -149,9 +119,6 @@ def build_inlinepatterns(md: Markdown, **kwargs: Any) -> util.Registry[InlinePro IMAGE_REFERENCE_RE = IMAGE_LINK_RE """ Match start of image reference (`![alt text][2]`). """ -NOT_STRONG_RE = r'((^|(?<=\s))(\*{1,3}|_{1,3})(?=\s|$))' -""" Match a stand-alone `*` or `_`. """ - AUTOLINK_RE = r'<((?:[Ff]|[Hh][Tt])[Tt][Pp][Ss]?://[^<>]*)>' """ Match an automatic link (``). """ @@ -592,151 +559,376 @@ def _unescape(m: re.Match[str]) -> str: return RE.sub(_unescape, text) -class AsteriskProcessor(InlineProcessor): - """Emphasis processor for handling strong and em matches inside asterisks.""" +class DelimiterProcessor(InlineProcessor): + """Processor for handling complex nested patterns such as strong and em matches.""" - PATTERNS = [ - EmStrongItem(re.compile(EM_STRONG_RE, re.DOTALL | re.UNICODE), 'double', 'strong,em'), - EmStrongItem(re.compile(STRONG_EM_RE, re.DOTALL | re.UNICODE), 'double', 'em,strong'), - EmStrongItem(re.compile(STRONG_EM3_RE, re.DOTALL | re.UNICODE), 'double2', 'strong,em'), - EmStrongItem(re.compile(STRONG_RE, re.DOTALL | re.UNICODE), 'single', 'strong'), - EmStrongItem(re.compile(EMPHASIS_RE, re.DOTALL | re.UNICODE), 'single', 'em') - ] - """ The various strong and emphasis patterns handled by this processor. """ - - def build_single(self, m: re.Match[str], tag: str, idx: int) -> etree.Element: - """Return single tag.""" - el1 = etree.Element(tag) - text = m.group(2) - self.parse_sub_patterns(text, el1, None, idx) - return el1 - - def build_double(self, m: re.Match[str], tags: str, idx: int) -> etree.Element: - """Return double tag.""" + def __init__( + self, + token: str, + tags: str, + md: Markdown, + smart: bool = False, + double: bool = False + ) -> None: + """ + Initialize. - tag1, tag2 = tags.split(",") - el1 = etree.Element(tag1) - el2 = etree.Element(tag2) - text = m.group(2) - self.parse_sub_patterns(text, el2, None, idx) - el1.append(el2) - if len(m.groups()) == 3: - text = m.group(3) - self.parse_sub_patterns(text, el1, el2, idx) - return el1 + Arguments: + token: A single character token. + tags: A tag or two tags seprated by comma. When two are specified, the first will be the + one that takes double tokens. + md: the Markdown object + smart: Enable intelligent word logic. + double: If only one tag is specified, indicate whether it requires double tokens. - def build_double2(self, m: re.Match[str], tags: str, idx: int) -> etree.Element: - """Return double tags (variant 2): `text text`.""" + """ - tag1, tag2 = tags.split(",") - el1 = etree.Element(tag1) - el2 = etree.Element(tag2) - text = m.group(2) - self.parse_sub_patterns(text, el1, None, idx) - text = m.group(3) - el1.append(el2) - self.parse_sub_patterns(text, el2, None, idx) - return el1 + # Cache info + md.delimiters[token] = self + self.regions: list[tuple[int, int, int, int, int]] = [] + self.stack: deque[tuple[int, int, bool, int]] = deque() + self.cache_index = 0 + self.cache_pos = 0 + + self.last_run = 0.0 + self.smart = smart + self.tags = tags.split(',') + self.double = len(tags) != 2 and double + super().__init__(self._build_patterns(token), md) + + def reset(self) -> None: + """Rest.""" + + self.regions.clear() + self.stack.clear() + self.cache_index = 0 + self.cache_pos = 0 + + def _build_patterns(self, token: str) -> str: + """Build regular expression patterns.""" + + # Build up patterns + self.token = token + etoken = re.escape(token) + avoid_start = fr'(?:(?<=_)|(?(?(?{avoid_start}{etoken}{{{n}}}(?![\s{etoken}])(?!$)) + ''', + flags=re.UNICODE + ) + # Patterns for "dumb" cases. + else: + self.boundary = re.compile( + fr'''(?x)(?: + (?P(?(?{etoken}{{{n}}}(?![\s{etoken}])(?!$)) + )''', + flags=re.UNICODE + ) + + return fr'{etoken}' + + def _build_element( + self, + data: str, + start: int = 0, + offset: int = 0 + ) -> tuple[etree.Element, int]: + """Element builder.""" - def parse_sub_patterns( - self, data: str, parent: etree.Element, last: etree.Element | None, idx: int - ) -> None: - """ - Parses sub patterns. + regions = self.regions + el: etree.Element | None = None + last: Any = None + previous: Any = None + greater: Any = None + lesser: Any = None + + triple = set() + outer: list[etree.Element] = [] + outer_r: list[tuple[int, int, int, int, int]] = [] + + if len(self.tags) == 2: + greater, lesser = self.tags + elif self.double: # pragma: no cover + greater = self.tags[0] + lesser = None + else: # pragma: no cover + lesser = self.tags[0] + greater = None + + # Iterate regions creating the elements they represent + end = len(regions) + idx = 0 + for idx, i in enumerate(range(start, end), 1): + r = regions[i] + # Not contained within region + if idx and r[0] >= regions[start][3]: + idx -= 1 + break + # Get the appropriate element(s) + if r[4] == 3: + el1 = etree.Element(lesser) + el2 = etree.Element(greater) + elif r[4] == 2: + el1 = etree.Element(greater) + el2 = None + else: + el1 = etree.Element(lesser) + el2 = None + + # Populate the elements with their text + if idx > 1: + if last.text is None: + if previous[2] < r[0]: + last.text = data[previous[1]+offset:previous[2]+offset] + else: + last.text = data[previous[1]+offset:r[0]+offset] + if last is not outer[-1] and last.tail is None: + if r[0] < outer_r[-1][3]: + last.tail = data[previous[3]+offset:r[0]+offset] + else: + last.tail = data[previous[3]+offset:outer_r[-1][2]+offset] + outer[-1].tail = data[outer_r[-1][3]+offset:r[0]+offset] - `data`: text to evaluate. + # First element + if el is None: + el = el1 + last = el + outer.append(el) + outer_r.append(r) - `parent`: Parent to attach text and sub elements to. + # Subsequent elements + else: + # Is the current outer element no longer wrapping this one? + while len(outer_r) > 1 and r[3] > outer_r[-1][3]: + outer.pop() + outer_r.pop() - `last`: Last appended child to parent. Can also be None if parent has no children. + # Double nested element (triple token) + if outer[-1] in triple: + outer[-1][-1].append(el1) - `idx`: Current pattern index that was used to evaluate the parent. + # Non-nested + else: + outer[-1].append(el1) + + # Is this element wrapping the next? + if i + 1 < end: + if r[3] > regions[i + 1][3]: + outer.append(el1) + outer_r.append(r) + + # Track the last element we parsed. + last = el1 + + # Nest secondary element if there is one. + # Track triple tokens (double elements) + # so we can identify quickly and properly nest. + if el2 is not None: + el1.append(el2) + last = el2 + triple.add(el1) + + # Track the previous region. + previous = r + + # Populate remaining elements with their text + while outer: + if last.text is None: + last.text = data[previous[1]+offset:previous[2]+offset] + if last.tail is None and last is not outer[-1]: + last.tail = data[previous[3]+offset:outer_r[-1][2]+offset] + last = outer.pop() + previous = outer_r.pop() + + return cast('etree.Element', el), idx + + def increment_next_position(self, start: int, count: int) -> None: """ + Increment cache position to the next location that we can initiate an insertion. - offset = 0 - pos = 0 - - length = len(data) - while pos < length: - # Find the start of potential emphasis or strong tokens - if self.compiled_re.match(data, pos): - matched = False - # See if the we can match an emphasis/strong pattern - for index, item in enumerate(self.PATTERNS): - # Only evaluate patterns that are after what was used on the parent - if index <= idx: - continue - m = item.pattern.match(data, pos) - if m: - # Append child nodes to parent - # Text nodes should be appended to the last - # child if present, and if not, it should - # be added as the parent's text node. - text = data[offset:m.start(0)] - if text: - if last is not None: - last.tail = text - else: - parent.text = text - el = self.build_element(m, item.builder, item.tags, index) - parent.append(el) - last = el - # Move our position past the matched hunk - offset = pos = m.end(0) - matched = True - if not matched: - # We matched nothing, move on to the next character - pos += 1 - else: - # Increment position as no potential emphasis start was found. - pos += 1 - - # Append any leftover text as a text node. - text = data[offset:] - if text: - if last is not None: - last.tail = text - else: - parent.text = text + Cache position should be the first match after our current replacement. + This gives us an anchor to calculate the new offset after insertion. + """ - def build_element(self, m: re.Match[str], builder: str, tags: str, index: int) -> etree.Element: - """Element builder.""" + # Determine next offset + self.cache_index += count + if self.cache_index < len(self.regions): + self.cache_pos = self.regions[self.cache_index][0] + while self.stack: + entry = self.stack.popleft() + if entry[0] > start: + self.cache_pos = entry[0] + break - if builder == 'double2': - return self.build_double2(m, tags, index) - elif builder == 'double': - return self.build_double(m, tags, index) + # Nothing left to process else: - return self.build_single(m, tags, index) + self.reset() + + def get_cached_result(self, pos: int, data: str) -> tuple[etree.Element, int, int]: + """Get a cached result.""" + + # Process the next region(s) in the cache + regions = self.regions + offset = pos - self.cache_pos if pos != self.cache_pos else pos - regions[self.cache_index][0] + start, end = regions[self.cache_index][0], regions[self.cache_index][3] + el, count = self._build_element(data, self.cache_index, offset) + self.increment_next_position(start, count) + return el, start + offset, end + offset + + def handleMatch( # type: ignore[override] + self, + m: re.Match[str], + data: str + ) -> tuple[etree.Element | None, int | None, int | None]: + """Parse delimiter pattern.""" + + # Do we have entries we haven't returned yet? + if self.regions: + return self.get_cached_result(m.start(0), data) + + # If token is not an opening, quit + m2 = self.boundary.match(data, m.start(0)) + if m2 is None or m2.lastgroup[0] == 'e': # type: ignore[index] + if m2 is not None: + m = m2 + # Advance past the full length of the delimiter found + return None, m.start(0), m.end(0) - def handleMatch(self, m: re.Match[str], data: str) -> tuple[etree.Element | None, int | None, int | None]: - """Parse patterns.""" - - el = None - start = None - end = None - - for index, item in enumerate(self.PATTERNS): - m1 = item.pattern.match(data, m.start(0)) - if m1: - start = m1.start(0) - end = m1.end(0) - el = self.build_element(m1, item.builder, item.tags, index) - break - return el, start, end + # Get the stack and regions + stack = self.stack + regions = self.regions + start = m2.start(0) + end = m2.end(0) + length = end - start + is_ambiguous = m2.lastgroup[0] != 's' # type: ignore[index] + self.stack.append((start, start + length, is_ambiguous, length)) -class UnderscoreProcessor(AsteriskProcessor): - """Emphasis processor for handling strong and em matches inside underscores.""" + # Pair tokens until the stack is empty or we can no longer find tokens. + while stack: + m2 = self.boundary.search(data, end) + if m2 is None: + break + start = m2.start(0) + end = m2.end(0) + + # Get current and last delimiter size + current = len(m2.group(0)) + last = stack[-1][-1] + + # Some delimiters may be ambiguous and look like both a start or an end + is_start = m2.lastgroup[0] != 'e' # type: ignore[index] + is_end = not is_start or m2.lastgroup[0] != 's' # type: ignore[index] + is_ambiguous = is_start and is_end + + # Find closing tokens + # Looking for: + # - `*em*` + # - `**strong**` + # - `***strong,em***` + # - `*em**` + # - `*em***` + # - `**strong***` + # + # Avoid ambiguous tokens that could be a start or an end. + # Consume starts until the end token is fully consumed. + # If we don't consume the entire end, see if next rule consumes it. + if ( + is_end and + ((not is_ambiguous and current > last) or (current <= 3 and current == last) or current >= 3) + ): + is_start = False + + # Consume previous tokens until the delimiter is consumed + original = current + while current and last <= current: + delimiter = stack.pop() + + # Build up region for pair and adjust accounting. + size = min(delimiter[-1], 2) + regions.append((delimiter[1] - size, delimiter[1], start, start + size, size)) + start += size + current -= size + if size < delimiter[-1]: + new = delimiter[-1] - size + stack.append((delimiter[0], delimiter[1] - size, delimiter[2], new)) + if not stack: + is_end = False + break + last = stack[-1][-1] + + # Should remainder be treated as a new start? + if original >= 3 and current and is_ambiguous: + self.stack.append((regions[-1][3], end, False, current)) + is_end = False + + # Do we still have more to consume? + else: + is_end = current and stack and last > current + + # Looking for: + # - `***em*` + # - `***strong**` + # - `**em*` + if is_end and (last >= 3 or not is_ambiguous) and last > current: + delimiter = stack.pop() + + # Don't pair with an ambiguous opening + while stack and delimiter[-1] != 3 and delimiter[2]: + delimiter = stack.pop() + last = delimiter[-1] + if delimiter[2]: + break - PATTERNS = [ - EmStrongItem(re.compile(EM_STRONG2_RE, re.DOTALL | re.UNICODE), 'double', 'strong,em'), - EmStrongItem(re.compile(STRONG_EM2_RE, re.DOTALL | re.UNICODE), 'double', 'em,strong'), - EmStrongItem(re.compile(SMART_STRONG_EM_RE, re.DOTALL | re.UNICODE), 'double2', 'strong,em'), - EmStrongItem(re.compile(SMART_STRONG_RE, re.DOTALL | re.UNICODE), 'single', 'strong'), - EmStrongItem(re.compile(SMART_EMPHASIS_RE, re.DOTALL | re.UNICODE), 'single', 'em') - ] - """ The various strong and emphasis patterns handled by this processor. """ + is_start = False + ds, de = delimiter[:2] + while current: + size = min(current, 2) + new = last - size + regions.append((ds + new, de, start, start + size, size)) + start += size + current -= size + last -= size + de -= size + stack.append((ds, de, False, last)) + + # Find opening tokens + if is_start: + # Looking for: + # - `*em ...*` + # - `**strong ...*` + # - `***em ...*` + stack.append((start, end, is_ambiguous, current)) + + # Build the HTML elements + if regions: + # Regions may be out of order. + regions.sort(key=lambda x: x[0]) + start, end = regions[0][0], regions[0][3] + el, count = self._build_element(data) + self.increment_next_position(start, count) + return el, start, end + + # We failed to pair any valid start/end delimiters, avoid the parsed range next pass. + start = m.start(0) + end = stack[-1][1] if stack else m.end(0) + self.reset() + return None, start, end class LinkInlineProcessor(InlineProcessor): diff --git a/tests/basic/strong-and-em-together.html b/tests/basic/strong-and-em-together.html deleted file mode 100644 index 7bf5163e7..000000000 --- a/tests/basic/strong-and-em-together.html +++ /dev/null @@ -1,4 +0,0 @@ -

This is strong and em.

-

So is this word.

-

This is strong and em.

-

So is this word.

\ No newline at end of file diff --git a/tests/basic/strong-and-em-together.txt b/tests/basic/strong-and-em-together.txt deleted file mode 100644 index 95ee690db..000000000 --- a/tests/basic/strong-and-em-together.txt +++ /dev/null @@ -1,7 +0,0 @@ -***This is strong and em.*** - -So is ***this*** word. - -___This is strong and em.___ - -So is ___this___ word. diff --git a/tests/misc/em_strong_complex.html b/tests/misc/em_strong_complex.html deleted file mode 100644 index 65faddfad..000000000 --- a/tests/misc/em_strong_complex.html +++ /dev/null @@ -1,14 +0,0 @@ -

test test test test

-

test test test test

-

test

-

test

-

test test_

-

test test

-

test_test test_test

-

test test test test

-

test test test test

-

*test

-

test

-

test*

-

test test

-

testtest testtest

\ No newline at end of file diff --git a/tests/misc/em_strong_complex.txt b/tests/misc/em_strong_complex.txt deleted file mode 100644 index 042597184..000000000 --- a/tests/misc/em_strong_complex.txt +++ /dev/null @@ -1,27 +0,0 @@ -___test test__ test test_ - -___test test_ test test__ - -___test___ - -__test__ - -___test_ test___ - -___test_ test__ - -_test_test test_test_ - -***test test** test test* - -***test test* test test** - -**test* - -***test*** - -**test*** - -***test* test** - -*test*test test*test* \ No newline at end of file diff --git a/tests/misc/nested-patterns.html b/tests/misc/nested-patterns.html deleted file mode 100644 index 1c7bb43c6..000000000 --- a/tests/misc/nested-patterns.html +++ /dev/null @@ -1,10 +0,0 @@ -

link -link -link -link -link -link -link

-

I am italic and bold I am just bold

-

Example bold italic on the same line bold italic.

-

Example bold italic on the same line bold italic.

\ No newline at end of file diff --git a/tests/misc/nested-patterns.txt b/tests/misc/nested-patterns.txt deleted file mode 100644 index 9032cf131..000000000 --- a/tests/misc/nested-patterns.txt +++ /dev/null @@ -1,13 +0,0 @@ -___[link](http://example.com)___ -***[link](http://example.com)*** -**[*link*](http://example.com)** -__[_link_](http://example.com)__ -__[*link*](http://example.com)__ -**[_link_](http://example.com)** -[***link***](http://example.com) - -***I am ___italic_ and__ bold* I am `just` bold** - -Example __*bold italic*__ on the same line __*bold italic*__. - -Example **_bold italic_** on the same line **_bold italic_**. diff --git a/tests/misc/underscores.html b/tests/misc/underscores.html deleted file mode 100644 index 72d51b8b5..000000000 --- a/tests/misc/underscores.html +++ /dev/null @@ -1,6 +0,0 @@ -

THIS_SHOULD_STAY_AS_IS

-

Here is some emphasis, ok?

-

Ok, at least this should work.

-

THIS__SHOULD__STAY

-

Here is some strong stuff.

-

THISSHOULDSTAY?

\ No newline at end of file diff --git a/tests/misc/underscores.txt b/tests/misc/underscores.txt deleted file mode 100644 index 3c7f4bdd9..000000000 --- a/tests/misc/underscores.txt +++ /dev/null @@ -1,11 +0,0 @@ -THIS_SHOULD_STAY_AS_IS - -Here is some _emphasis_, ok? - -Ok, at least _this_ should work. - -THIS__SHOULD__STAY - -Here is some __strong__ stuff. - -THIS___SHOULD___STAY? diff --git a/tests/test_legacy.py b/tests/test_legacy.py index df4da4d71..808def4c6 100644 --- a/tests/test_legacy.py +++ b/tests/test_legacy.py @@ -91,7 +91,7 @@ class TestPl2004(LegacyTestCase): location = os.path.join(parent_test_dir, 'pl/Tests_2004') normalize = True input_ext = '.text' - exclude = ['Yuri_Footnotes', 'Yuri_Attributes'] + exclude = ['Yuri_Footnotes', 'Yuri_Attributes', 'Strong_and_em_together'] class TestPl2007(LegacyTestCase): @@ -116,7 +116,8 @@ class TestPl2007(LegacyTestCase): 'Code_Blocks', 'Links,_reference_style', 'Backslash_escapes', - 'Code_Spans' + 'Code_Spans', + 'Strong_and_em_together' ] diff --git a/tests/test_syntax/blocks/test_hr.py b/tests/test_syntax/blocks/test_hr.py index a8753977b..5aa48465b 100644 --- a/tests/test_syntax/blocks/test_hr.py +++ b/tests/test_syntax/blocks/test_hr.py @@ -318,7 +318,7 @@ def test_hr_after_emstrong(self): ), self.dedent( """ -

text

+

text

""" ) diff --git a/tests/test_syntax/extensions/test_legacy_em.py b/tests/test_syntax/extensions/test_legacy_em.py index ad2862e5e..a1a9f2e5a 100644 --- a/tests/test_syntax/extensions/test_legacy_em.py +++ b/tests/test_syntax/extensions/test_legacy_em.py @@ -55,12 +55,12 @@ def test_complex_multple_underscore_type(self): self.assertMarkdownRenders( 'traced ___along___ bla __blocked__ if other ___or___', - '

traced along bla blocked if other or

' # noqa: E501 + '

traced along bla blocked if other or

' # noqa: E501 ) def test_complex_multple_underscore_type_variant2(self): self.assertMarkdownRenders( 'on the __1-4 row__ of the AP Combat Table ___and___ receive', - '

on the 1-4 row of the AP Combat Table and receive

' + '

on the 1-4 row of the AP Combat Table and receive

' ) diff --git a/tests/test_syntax/inline/test_emphasis.py b/tests/test_syntax/inline/test_emphasis.py index 6e96ea32c..127d2c78f 100644 --- a/tests/test_syntax/inline/test_emphasis.py +++ b/tests/test_syntax/inline/test_emphasis.py @@ -147,7 +147,7 @@ def test_complex_emphasis_smart_underscore(self): def test_complex_emphasis_smart_underscore_mid_word(self): self.assertMarkdownRenders( 'This is text __bold_italic bold___ with more text', - '

This is text __bold_italic bold___ with more text

' + '

This is text bold_italic bold_ with more text

' ) def test_nested_emphasis(self): @@ -161,14 +161,14 @@ def test_complex_multple_emphasis_type(self): self.assertMarkdownRenders( 'traced ***along*** bla **blocked** if other ***or***', - '

traced along bla blocked if other or

' # noqa: E501 + '

traced along bla blocked if other or

' # noqa: E501 ) def test_complex_multple_emphasis_type_variant2(self): self.assertMarkdownRenders( 'on the **1-4 row** of the AP Combat Table ***and*** receive', - '

on the 1-4 row of the AP Combat Table and receive

' + '

on the 1-4 row of the AP Combat Table and receive

' ) def test_link_emphasis_outer(self): @@ -191,3 +191,678 @@ def test_link_emphasis_inner_outer(self): '**[**text**](url)**', '

text

' ) + + def test_underscore_legacy(self): + + self.assertMarkdownRenders( + self.dedent( + """ + THIS_SHOULD_STAY_AS_IS + + Here is some _emphasis_, ok? + + Ok, at least _this_ should work. + + THIS__SHOULD__STAY + + Here is some __strong__ stuff. + + THIS___SHOULD___STAY? + """ + ), + self.dedent( + """ +

THIS_SHOULD_STAY_AS_IS

+

Here is some emphasis, ok?

+

Ok, at least this should work.

+

THIS__SHOULD__STAY

+

Here is some strong stuff.

+

THIS___SHOULD___STAY?

+ """ + ) + ) + + def test_nested_patterns(self): + + self.assertMarkdownRenders( + self.dedent( + """ + ___[link](http://example.com)___ + ***[link](http://example.com)*** + **[*link*](http://example.com)** + __[_link_](http://example.com)__ + __[*link*](http://example.com)__ + **[_link_](http://example.com)** + [***link***](http://example.com) + + ***I am ___italic_ and__ bold* I am `just` bold** + + Example __*bold italic*__ on the same line __*bold italic*__. + + Example **_bold italic_** on the same line **_bold italic_**. + """ + ), + self.dedent( + """ +

link + link + link + link + link + link + link

+

I am italic and bold I am just bold

+

Example bold italic on the same line bold italic.

+

Example bold italic on the same line bold italic.

+ """ # noqa: E501 + ) + ) + + def test_em_strong_complex(self): + + self.assertMarkdownRenders( + self.dedent( + """ + ___test test__ test test_ + + ___test test_ test test__ + + ___test___ + + __test__ + + ___test_ test___ + + ___test_ test__ + + _test_test test_test_ + + ***test test** test test* + + ***test test* test test** + + **test* + + ***test*** + + **test*** + + ***test* test** + + *test*test test*test* + """ + ), + self.dedent( + """ +

test test test test

+

test test test test

+

test

+

test

+

test test_

+

test test

+

test_test test_test

+

test test test test

+

test test test test

+

*test

+

test

+

test*

+

test test

+

testtest testtest

+ """ + ) + ) + + def test_strong_and_em_together(self): + + self.assertMarkdownRenders( + self.dedent( + """ + ***This is strong and em.*** + + So is ***this*** word. + + ___This is strong and em.___ + + So is ___this___ word. + """ + ), + self.dedent( + """ +

This is strong and em.

+

So is this word.

+

This is strong and em.

+

So is this word.

+ """ + ) + ) + + def test_advanced_nesting(self): + + self.assertMarkdownRenders( + self.dedent( + """ + **a*bc** + + *a**b**c**d**e**f* + + ***a**b*cd**e*f*** + + ***a**b*cd*e**f*** + + ***a**b*cd*e**f*g*h*** + + ***a***bc**d*e*** + + *a**b**c**d**e**f* + + *a**b***c**d***e**f* + + *a**b***c**d***e**f** + + __a _b c__ + + _a __b __c __d __e __f_ + + ___a __b _c d__ e_ f___ + + ___a __b _c d_ e__ f___ + + ___a __b _c d_ e__ f _g_ h___ + + ___a ___b c__ d_ e___ + + _a __b__ _c __d__ _e __f__ + """ + ), + self.dedent( + """ +

*abc

+

abcde**f

+

abcdef

+

abcdef

+

abcdefgh

+

abcde

+

abcde**f

+

abcde**f

+

abcd*ef

+

_a b c

+

_a __b __c __d __e _f

+

a b c d e f

+

a b c d e f

+

a b c d e f g h

+

a b c d e

+

_a b _c d _e f

+ """ + ) + ) + + +class TestCommonMark(TestCase): + """Test CommonMark.""" + + def test_commonmark(self): + """Test CommonMark.""" + + self.maxDiff = None + + self.assertMarkdownRenders( + self.dedent( + R""" + *foo bar* + + a * foo bar* + + + + + + + test * a * + + foo*bar* + + 5*6*78 + + _foo bar_ + + _ foo bar_ + + a_"foo"_ + + foo_bar_ + + 5_6_78 + + пристаням_стремятся_ + + aa_"bb"_cc + + foo-_(bar)_ + + _foo* + + *foo bar * + + *foo bar + * + + + + *(*foo*)* + + *foo*bar + + _foo bar _ + + _(_foo) + + _(_foo_)_ + + _foo_bar + + _пристаням_стремятся + + _foo_bar_baz_ + + _(bar)_. + + **foo bar** + + ** foo bar** + + + + foo**bar** + + __foo bar__ + + __ foo bar__ + + __ + foo bar__ + + a__"foo"__ + + foo__bar__ + + 5__6__78 + + пристаням__стремятся__ + + __foo, __bar__, baz__ + + foo-__(bar)__ + + **foo bar ** + + + + *(**foo**)* + + **Gomphocarpus (*Gomphocarpus physocarpus*, syn. + *Asclepias physocarpa*)** + + **foo "*bar*" foo** + + **foo**bar + + __foo bar __ + + __(__foo) + + _(__foo__)_ + + __foo__bar + + __пристаням__стремятся + + __foo__bar__baz__ + + __(bar)__. + + *foo [bar](/url)* + + *foo + bar* + + _foo __bar__ baz_ + + _foo _bar_ baz_ + + __foo_ bar_ + + *foo *bar** + + *foo **bar** baz* + + *foo**bar**baz* + + *foo**bar* + + ***foo** bar* + + *foo **bar*** + + *foo**bar*** + + foo***bar***baz + + foo******bar*********baz + + *foo **bar *baz* bim** bop* + + *foo [*bar*](/url)* + + ** is not an empty emphasis + + **** is not an empty strong emphasis + + **foo [bar](/url)** + + **foo + bar** + + __foo _bar_ baz__ + + __foo __bar__ baz__ + + ____foo__ bar__ + + **foo **bar**** + + **foo *bar* baz** + + **foo*bar*baz** + + ***foo* bar** + + **foo *bar*** + + **foo *bar **baz** + bim* bop** + + **foo [*bar*](/url)** + + __ is not an empty emphasis + + ____ is not an empty strong emphasis + + foo *** + + foo *\** + + foo *_* + + foo ***** + + foo **\*** + + foo **_** + + **foo* + + *foo** + + ***foo** + + ****foo* + + **foo*** + + *foo**** + + foo ___ + + foo _\__ + + foo _*_ + + foo _____ + + foo __\___ + + foo __*__ + + __foo_ + + _foo__ + + ___foo__ + + ____foo_ + + __foo___ + + _foo____ + + **foo** + + *_foo_* + + __foo__ + + _*foo*_ + + ****foo**** + + ____foo____ + + ******foo****** + + ***foo*** + + _____foo_____ + + *foo _bar* baz_ + + + + **foo **bar baz** + + *foo *bar baz* + + *[bar*](/url) + + _foo [bar_](/url) + + * + + ** + + __ + + *a `*`* + + _a `_`_ + + **a + + __a + """ + ), + self.dedent( + """ +

foo bar

+

a * foo bar*

+ + + + + +

test * a *

+

foobar

+

5678

+

foo bar

+

_ foo bar_

+

a_"foo"_

+

foo_bar_

+

5_6_78

+

пристаням_стремятся_

+

aa_"bb"_cc

+

foo-(bar)

+

_foo*

+

*foo bar *

+

*foo bar + *

+ + +

(foo)

+

foobar

+

_foo bar _

+

_(_foo)

+

(foo)

+

_foo_bar

+

_пристаням_стремятся

+

foo_bar_baz

+

(bar).

+

foo bar

+

** foo bar**

+ + +

foobar

+

foo bar

+

__ foo bar__

+

__ + foo bar__

+

a__"foo"__

+

foo__bar__

+

5__6__78

+

пристаням__стремятся__

+

foo, bar, baz

+

foo-(bar)

+

**foo bar **

+ + +

(foo)

+

Gomphocarpus (Gomphocarpus physocarpus, syn. + Asclepias physocarpa)

+

foo "bar" foo

+

foobar

+

__foo bar __

+

__(__foo)

+

(foo)

+

__foo__bar

+

__пристаням__стремятся

+

foo__bar__baz

+

(bar).

+

foo bar

+

foo + bar

+

foo bar baz

+

foo bar baz

+

foo bar

+

foo bar

+

foo bar baz

+

foobarbaz

+

foo**bar

+

foo bar

+

foo bar

+

foobar

+

foobarbaz

+

foobar***baz

+

foo bar baz bim bop

+

foo bar

+

** is not an empty emphasis

+

**** is not an empty strong emphasis

+

foo bar

+

foo + bar

+

foo bar baz

+

foo bar baz

+

foo bar

+

foo bar

+

foo bar baz

+

foobarbaz

+

foo bar

+

foo bar

+

foo bar baz + bim bop

+

foo bar

+

__ is not an empty emphasis

+

____ is not an empty strong emphasis

+

foo ***

+

foo *

+

foo _

+

foo *****

+

foo *

+

foo _

+

*foo

+

foo*

+

*foo

+

***foo

+

foo*

+

foo***

+

foo ___

+

foo _

+

foo *

+

foo _____

+

foo _

+

foo *

+

_foo

+

foo_

+

_foo

+

___foo

+

foo_

+

foo___

+

foo

+

foo

+

foo

+

foo

+

foo

+

foo

+

foo

+

foo

+

foo

+

foo _bar baz_

+ + +

**foo bar baz

+

*foo bar baz

+

*bar*

+

_foo bar_

+

*

+

**

+

__

+

a *

+

a _

+

**ahttps://foo.bar/?q=**

+

__ahttps://foo.bar/?q=__

+ """ + ) + ) + + +class TestProcessorRemoval(TestCase): + + def test_remove_processor(self): + + import markdown + from markdown.inlinepatterns import DelimiterProcessor + + # Remove all delimiter processors + md = markdown.Markdown() + + extensions = md.delimiters.values() + self.assertEqual(len(extensions), 2) + + for ext in md.delimiters.values(): + self.assertTrue(isinstance(ext, DelimiterProcessor)) + + md.inlinePatterns.deregister('em_strong') + md.inlinePatterns.deregister('em_strong2') + + # Call reset which will cause them to remove themselves from being registered + md.reset() + + extensions = md.delimiters.values() + self.assertEqual(len(extensions), 0)