diff --git a/docs/changelog.md b/docs/changelog.md
index e137b8f5..81e55e19 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -12,6 +12,16 @@ and this project adheres to the
[Python Version Specification](https://packaging.python.org/en/latest/specifications/version-specifiers/).
See the [Contributing Guide](contributing.md) for details.
+## Unreleased
+
+### Fixed
+
+* Preserve unused closing emphasis delimiters in their original context and
+ pair rule-of-three matches with the selected opener, without dropping text.
+
+* Fix for complex cases in new emphasis handling where text could be unexpectedly
+ dropped (#1694).
+
## [3.11.3] - 2026-11-05
### Fixed
diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py
index 2f05a1f8..01102b97 100644
--- a/markdown/inlinepatterns.py
+++ b/markdown/inlinepatterns.py
@@ -46,6 +46,7 @@
import re
import xml.etree.ElementTree as etree
from html import entities
+from bisect import bisect_left
if TYPE_CHECKING: # pragma: no cover
from markdown import Markdown
@@ -559,6 +560,16 @@ def _unescape(m: re.Match[str]) -> str:
return RE.sub(_unescape, text)
+class EvaluatedText(str):
+ """
+ Text that sits between delimiter regions that were parsed together.
+
+ Python Markdown processes the tail of an element we return starting with the same pattern that created it
+ and pass it back to us while we still have regions cached. All delimiters in it have already been evaluated,
+ so this represents text we can ignore.
+ """
+
+
class Delimiter:
"""Delimiter."""
@@ -644,7 +655,13 @@ def __init__(
md.delimiters = self
self.regions: list[tuple[int, int, int, int, tuple[str, str], int]] = []
# Unpaired delimiter. The entries are the same as `Delimiter.stack`.
+ # HTML element cache of valid regions. Each entry holds the element, the location it replaces,
+ # and the location of the token Python Markdown will call us for once the element has been replaced.
+ self.cache: list[tuple[etree.Element, int, int, int]] = []
+ # The entries are the same as `Delimiter.stack`.
self.stack: list[tuple[int, int, bool, int, int]] = []
+ # Position of every delimiter token (valid or not) encountered while pairing, in ascending order.
+ self.positions: list[int] = []
self.tokens: list[str] = []
self.delimiters: dict[str, Delimiter] = {}
self.cache_index = 0
@@ -688,15 +705,16 @@ def reset(self) -> None:
for v in self.delimiters.values():
v.reset()
self.regions.clear()
+ self.cache.clear()
self.stack.clear()
+ self.positions.clear()
self.cache_index = 0
self.cache_pos = 0
def _build_element(
self,
data: str,
- start: int = 0,
- offset: int = 0
+ start: int = 0
) -> tuple[etree.Element, int]:
"""Element builder."""
@@ -728,15 +746,15 @@ def _build_element(
if idx > 1:
if last.text is None:
if previous[2] < r[0]:
- last.text = data[previous[1]+offset:previous[2]+offset]
+ last.text = data[previous[1]:previous[2]]
else:
- last.text = data[previous[1]+offset:r[0]+offset]
+ last.text = data[previous[1]:r[0]]
if last is not outer[-1] and last.tail is None:
if r[0] < outer_r[-1][3]:
- last.tail = data[previous[3]+offset:r[0]+offset]
+ last.tail = EvaluatedText(data[previous[3]:r[0]])
else:
- last.tail = data[previous[3]+offset:outer_r[-1][2]+offset]
- outer[-1].tail = data[outer_r[-1][3]+offset:r[0]+offset]
+ last.tail = EvaluatedText(data[previous[3]:outer_r[-1][2]])
+ outer[-1].tail = EvaluatedText(data[outer_r[-1][3]:r[0]])
# First element
if el is None:
@@ -771,45 +789,70 @@ def _build_element(
# Populate remaining elements with their text
while outer:
if last.text is None:
- last.text = data[previous[1]+offset:previous[2]+offset]
+ last.text = data[previous[1]:previous[2]]
if last.tail is None and last is not outer[-1]:
- last.tail = data[previous[3]+offset:outer_r[-1][2]+offset]
+ last.tail = EvaluatedText(data[previous[3]:outer_r[-1][2]])
last = outer.pop()
previous = outer_r.pop()
return cast('etree.Element', el), idx
- def increment_next_position(self, start: int, count: int) -> None:
+ def build_cache(self, data: str) -> None:
"""
- Increment cache position to the next location that we can initiate an insertion.
+ Build the elements for all the regions we found.
- Cache position should be the first match after our current replacement.
- This gives us an anchor to calculate the new offset after insertion.
+ Elements are built from the data as it is now, so there is nothing to recalculate when Python Markdown
+ asks for them.
+
+ While pairing, we encounter every token between the regions we found, whether it was useful or not.
+ Python Markdown will call us for each of them, so the call after an element is replaced will be for the first
+ token at or after the end of that element. If an element only used part of a delimiter run, the rest
+ of the run is that token.
"""
- # Determine next offset
- self.cache_index += count
- if self.cache_index < len(self.regions):
- self.cache_pos = self.regions[self.cache_index][0]
- while self.stack:
- entry = self.stack.pop(0)
- if start < entry[0] <= self.cache_pos:
- self.cache_pos = entry[0]
- break
+ regions = self.regions
+ index = 0
+ position = 0
+ while index < len(regions):
+ el, count = self._build_element(data, index)
+ start, end = regions[index][0], regions[index][3]
+ token = data[end - 1]
+ if token in self.delimiters and data[end:end + 1] == token:
+ run_start = end - 1
+ while run_start and data[run_start - 1] == token:
+ run_start -= 1
+ closing = self.delimiters[token].boundary.match(data, run_start)
+ if closing is not None and closing.lastgroup == 'end':
+ # A closing-only run cannot become an opener when its prefix
+ # is replaced by an inline placeholder. Keep its unused tokens
+ # literal instead of parsing them again in the changed context.
+ el.tail = util.AtomicString(data[end:closing.end()])
+ end = closing.end()
+ if data[end:end + 1] in self.delimiters:
+ anchor = end
+ else:
+ position = bisect_left(self.positions, end, position)
+ anchor = self.positions[position] if position < len(self.positions) else end
+ self.cache.append((el, start, end, anchor))
+ index += count
+ regions.clear()
+ self.positions.clear()
+ self.cache_index = 0
+
+ def get_cached_result(self, pos: int | None = None) -> tuple[etree.Element, int, int]:
+ """Get the next element from the cache."""
+
+ el, start, end, anchor = self.cache[self.cache_index]
+ offset = 0 if pos is None else pos - self.cache_pos
+ self.cache_index += 1
+
+ if self.cache_index < len(self.cache):
+ self.cache_pos = anchor
# Nothing left to process
else:
self.reset()
- def get_cached_result(self, pos: int, data: str) -> tuple[etree.Element, int, int]:
- """Get a cached result."""
-
- # Process the next region(s) in the cache
- regions = self.regions
- offset = pos - self.cache_pos if pos != self.cache_pos else pos - regions[self.cache_index][0]
- start, end = regions[self.cache_index][0], regions[self.cache_index][3]
- el, count = self._build_element(data, self.cache_index, offset)
- self.increment_next_position(start, count)
return el, start + offset, end + offset
@staticmethod
@@ -836,6 +879,7 @@ def search(self, data: str, start: int) -> re.Match[str] | None:
for i in range(start, len(data)):
if data[i] in self.delimiters:
+ self.positions.append(i)
d = self.delimiters[data[i]]
m = d.boundary.match(data, i)
if m is not None:
@@ -848,11 +892,12 @@ def add_region(self, delim: Delimiter, a: int, b: int, c: int, d: int, size: int
"""Add region."""
self.regions.append((a, b, c, d, delim.tags, size))
- for delim in self.delimiters.values():
- while delim.stack:
- p = delim.stack[-1][0]
+ # Unpaired delimiters inside the new region can no longer be used, drop them
+ for other in self.delimiters.values():
+ while other.stack:
+ p = other.stack[-1][0]
if max(p, a) <= min(p, d - 1):
- delim.stack.pop()
+ other.stack.pop()
continue
break
@@ -863,9 +908,15 @@ def handleMatch( # type: ignore[override]
) -> tuple[etree.Element | None, int | None, int | None]:
"""Parse delimiter pattern."""
+ # If the cache is not empty, it is possible to receive text that we already evaluated
+ # between found regions, ignore such text.
+ if self.cache and isinstance(data, EvaluatedText):
+ skip = self.delimiters[m.group(0)].bad.match(data, m.start(0))
+ return None, m.start(0), skip.end(0) if skip else m.end(0)
+
# Do we have entries we haven't returned yet?
- if self.regions:
- return self.get_cached_result(m.start(0), data)
+ if self.cache:
+ return self.get_cached_result(m.start(0))
# If token is not an opening, quit
m2 = self.get_match(data, m.start(0))
@@ -923,6 +974,10 @@ def handleMatch( # type: ignore[override]
):
for i in range(len(stack) - 2, -1, -1):
if stack[i][-1] == current and not self.rule_of_3(stack[i][3], current):
+ # Pair with the opener we found, not the rejected opener
+ # still at the top of the stack.
+ while len(stack) > i + 1:
+ stack.pop()
last = current
break
@@ -1026,10 +1081,8 @@ def handleMatch( # type: ignore[override]
if self.regions:
# Regions may be out of order.
self.regions.sort(key=lambda x: x[0])
- start, end = self.regions[0][0], self.regions[0][3]
- el, count = self._build_element(data)
- self.increment_next_position(start, count)
- return el, start, end
+ self.build_cache(data)
+ return self.get_cached_result()
# We failed to pair any valid start/end delimiters, avoid the parsed range next pass.
start = m.start(0)
diff --git a/tests/test_syntax/inline/test_emphasis.py b/tests/test_syntax/inline/test_emphasis.py
index 2b60353c..0f71bffb 100644
--- a/tests/test_syntax/inline/test_emphasis.py
+++ b/tests/test_syntax/inline/test_emphasis.py
@@ -387,7 +387,7 @@ def test_advanced_nesting(self):
abcde**f
abcdef
abcdef
- abcdefh***
+ abcde**fgh**
abcde
abcde**f
abcdef
@@ -831,85 +831,106 @@ def test_commonmark(self):
)
)
- def test_ambiguous_closer_skips_rejected_opener(self):
- """Test that an ambiguous closer can skip an opener rejected by the rule of 3."""
+ def test_ambiguous_closer_skips_rejected_opener(self):
+ """Test that an ambiguous closer can skip an opener rejected by the rule of 3."""
- self.assertMarkdownRenders(
- '__a "_b"__: x',
- 'a "_b": x
'
- )
+ self.assertMarkdownRenders(
+ '__a "_b"__: x',
+ 'a "_b": x
'
+ )
- def test_ambiguous_closer_skips_rejected_opener_asterisk(self):
- """Test the rule of 3 with asterisks."""
+ def test_ambiguous_closer_skips_rejected_opener_asterisk(self):
+ """Test the rule of 3 with asterisks."""
- self.assertMarkdownRenders(
- '**a "*b"**: x',
- 'a "*b": x
'
- )
+ self.assertMarkdownRenders(
+ '**a "*b"**: x',
+ 'a "*b": x
'
+ )
- def test_attribute_list_like_content(self):
- """Test the rule of 3 with a link followed by attribute list like content."""
+ def test_attribute_list_like_content(self):
+ """Test the rule of 3 with a link followed by attribute list like content."""
- self.assertMarkdownRenders(
- '__[A](u){target="_blank"}__: x',
- 'A{target="_blank"}: x
'
- )
+ self.assertMarkdownRenders(
+ '__[A](u){target="_blank"}__: x',
+ 'A{target="_blank"}: x
'
+ )
- def test_rejected_opener_is_original_run_length(self):
- """Test that the original length of a partially consumed run is used for the rule of 3."""
+ def test_rejected_opener_is_original_run_length(self):
+ """Test that the original length of a partially consumed run is used for the rule of 3."""
- self.assertMarkdownRenders(
- '__a_ __"_: x',
- '_a __"_: x
'
- )
+ self.assertMarkdownRenders(
+ '__a_ __"_: x',
+ '_a __"_: x
'
+ )
- def test_unambiguous_closer_unaffected(self):
- """Test that an unambiguous closer is not subject to the rule of 3."""
+ def test_unambiguous_closer_unaffected(self):
+ """Test that an unambiguous closer is not subject to the rule of 3."""
- self.assertMarkdownRenders(
- '__a "_b" c__ x',
- '_a "b" c x
'
- )
+ self.assertMarkdownRenders(
+ '__a "_b" c__ x',
+ '_a "b" c x
'
+ )
- def test_remainder_classified_by_original_context(self):
- """Test that the unused part of a closing run is classified by what preceded the run."""
+ def test_remainder_classified_by_original_context(self):
+ """Test that the unused part of a closing run is classified by what preceded the run."""
- self.check_markdown(
- '_a__"__"_: x',
- 'a_"__"_: x
'
- )
+ self.assertMarkdownRenders(
+ '_a__"__"_: x',
+ 'a_"__"_: x
'
+ )
- def test_remainder_classified_by_original_context_three(self):
- """Test the unused part of a longer closing run."""
+ def test_remainder_classified_by_original_context_three(self):
+ """Test the unused part of a longer closing run."""
- self.check_markdown(
- '_a___"_"__: x',
- 'a__"_"__: x
'
- )
+ self.assertMarkdownRenders(
+ '_a___"_"__: x',
+ 'a__"_"__: x
'
+ )
- def test_placeholder_treated_as_punctuation(self):
- """Test that a placeholder from a different delimiter is still treated as punctuation."""
+ def test_placeholder_treated_as_punctuation(self):
+ """Test that a placeholder from a different delimiter is still treated as punctuation."""
- self.check_markdown(
- '*test*_"test"_',
- 'test"test"
'
- )
+ self.assertMarkdownRenders(
+ '*test*_"test"_',
+ 'test"test"
'
+ )
- def test_ambiguous_closer_pairs_with_partially_consumed_opener(self):
- """Test that the rule of 3 uses the original run length of a partially consumed opener."""
+ def test_rule_of_three_with_relaxed_punctuation(self):
+ """The inner star is ambiguous under Python-Markdown's relaxed punctuation rules."""
- self.check_markdown(
- '**a*_**: x',
- 'a_*: x
'
- )
+ self.assertMarkdownRenders(
+ '**a*_**: x',
+ 'a*_: x
'
+ )
- def test_ambiguous_closer_rejected_by_original_run_length(self):
- """Test that the rule of 3 still rejects a pairing based on the original run lengths."""
+ def test_rule_of_three_keeps_rejected_inner_delimiter_literal(self):
+ """The inner star cannot pair with either two-star run (CommonMark rule 9)."""
- self.check_markdown(
- '**a*b**: x',
- '*ab: x
'
- )
+ self.assertMarkdownRenders(
+ '**a*b**: x',
+ 'a*b: x
'
+ )
+
+ def test_stray_token_before_cached_region(self):
+ """Test that a stray token between cached regions does not corrupt the cached region's offset."""
+
+ self.assertMarkdownRenders(
+ '***l* * **e**',
+ '**l * e
'
+ )
+
+ self.assertMarkdownRenders(
+ '**l* * **e**',
+ '*l * e
'
+ )
+
+ def test_unused_tokens_in_returned_element_tail(self):
+ """Test tokens inside an element's tail are not processed against regions cached for the parent text."""
+
+ self.assertMarkdownRenders(
+ '__*)a.* *ab_a_ _)_(*',
+ '_)a. *ab_a )(*
'
+ )
class TestProcessorRemoval(TestCase):