[pull] main from python:main

InternalDocs/parser.md

@@ -563,6 +563,23 @@ in the generated C parse code that allows to measure how much each rule uses
 memoization (check the [`Parser/pegen.c`](../Parser/pegen.c)
 file for more information) but it needs to be manually activated.
 
+The C generator also reuses memoized prefixes within consecutive alternatives.
+For example, in `prefix ':' NAME | prefix ':' NUMBER`, failure after the first
+`':'` normally requires another call to `prefix` and another memo lookup. The
+generated code can keep the result and ending position in local variables and
+reuse them when trying the next alternative.
+
+This applies only when the shared first item is a memoized rule, including a
+left-recursion leader, that the generator can prove consumes input on success.
+The locals are reset on each rule-body invocation, including each seed-growing
+iteration. Alternative order, cuts, and suffix backtracking are preserved. When
+`call_invalid_rules` is enabled, the generated code uses the original rule calls.
+
+The consumption analysis follows grammar items; it cannot inspect arbitrary C
+actions. As with memoization, actions must not invalidate cached results. In
+particular, suffix actions must not move the parser before their starting mark,
+rewrite buffered input, or replace memo entries for earlier positions.
+
 Automatic variables
 -------------------
 

Lib/pdb.py

@@ -2440,9 +2440,13 @@ def _print_lines(self, lines, start, breaks=(), frame=None):
                 s += '->'
             elif lineno == exc_lineno:
                 s += '>>'
+            # Strip the trailing newline before colorizing: the colorizer
+            # renders control characters (like '\n') in caret notation, so a
+            # later rstrip() could not remove the resulting '^J'.
+            line = line.rstrip()
             if self.colorize:
                 line = self._colorize_code(line)
-            self.message(s + '\t' + line.rstrip())
+            self.message(s + '\t' + line)
 
     def do_whatis(self, arg):
         """whatis expression

Lib/test/test_pdb.py

@@ -4996,6 +4996,16 @@ def test_code_display(self):
         p.set_trace(commands=['ll', 'c'])
         self.assertNotIn("\x1b", output.getvalue())
 
+    def test_list_does_not_colorize_trailing_newlines(self):
+        # Keep the marker split so it is not present in the listed source.
+        caret_newline = "^" + "J"
+        output = io.StringIO()
+        p = pdb.Pdb(stdout=output, colorize=True)
+        p.set_trace(commands=['list', 'continue'])
+        result = output.getvalue()
+        self.assertIn("\x1b", result)
+        self.assertNotIn(caret_newline, result)
+
     def test_stack_entry(self):
         output = io.StringIO()
         p = pdb.Pdb(stdout=output, colorize=True)

Lib/test/test_peg_generator/test_c_parser.py

@@ -145,6 +145,35 @@ def run_test(self, grammar_source, test_source):
             TEST_TEMPLATE.format(extension_path=self.tmp_path, test_source=test_source),
         )
 
+    def test_prefix_reuses_position(self) -> None:
+        grammar_source = """
+        start:
+            | prefix ':' NAME NEWLINE? ENDMARKER
+            | prefix ':' NUMBER NEWLINE? ENDMARKER
+            | prefix '=' NUMBER NEWLINE? ENDMARKER
+        prefix (memo): NAME NAME
+        """
+        self.run_test(grammar_source, """
+        self.check_input_strings_for_grammar(
+            valid_cases=['one two : name', 'one two : 3', 'one two = 3'],
+            invalid_cases=['one = 3', 'one two = name', 'one two :'],
+        )
+        """)
+
+    def test_prefix_respects_cut(self) -> None:
+        grammar_source = """
+        start:
+            | prefix ':' ~ NAME NEWLINE? ENDMARKER
+            | prefix ':' NUMBER NEWLINE? ENDMARKER
+        prefix (memo): NAME NAME
+        """
+        self.run_test(grammar_source, """
+        self.check_input_strings_for_grammar(
+            valid_cases=['one two : name'],
+            invalid_cases=['one two : 3'],
+        )
+        """)
+
     def test_c_parser(self) -> None:
         grammar_source = """
         start[mod_ty]: a[asdl_stmt_seq*]=stmt* $ { _PyAST_Module(a, NULL, p->arena) }

Lib/test/test_peg_generator/test_prefix.py

@@ -0,0 +1,33 @@
+import unittest
+
+from test import test_tools
+
+test_tools.skip_if_missing("peg_generator")
+with test_tools.imports_under_tool("peg_generator"):
+    from pegen.c_generator import consuming_rules
+    from pegen.testutil import GrammarParser, parse_string
+
+
+class ConsumingRuleTests(unittest.TestCase):
+    def test_predicates_cuts_and_nullable_repeats(self):
+        grammar = parse_string("""
+        start: NAME ENDMARKER
+        positive: &NAME
+        negative: !NAME
+        cut: ~ { _PyPegen_dummy_name(p) }
+        optional: [NAME]
+        empty_repeat: NAME*
+        nullable_repeat: optional+
+        consuming_repeat: NAME+
+        """, GrammarParser)
+        self.assertEqual(consuming_rules(grammar.rules), {'start', 'consuming_repeat'})
+
+    def test_fixed_point_and_mixed_alternatives(self):
+        grammar = parse_string("""
+        start: expression ENDMARKER
+        expression: expression '+' term | term
+        term: atom
+        atom: NAME | '(' expression ')'
+        nullable: NAME | &NAME
+        """, GrammarParser)
+        self.assertEqual(consuming_rules(grammar.rules), {'start', 'expression', 'term', 'atom'})

Lib/test/test_remote_pdb.py

@@ -1337,6 +1337,55 @@ def test_handle_eof(self):
             self.assertEqual(process.returncode, 0)
             self.assertEqual(stderr, "")
 
+    def test_colorized_list_has_no_caret_encoded_newlines(self):
+        """A colorized ``list`` must not append "^J" to each source line.
+
+        The remote server colorizes the source it sends to the client. The
+        colorizer renders control characters in caret notation, so a source
+        line's trailing newline has to be stripped *before* it is colorized;
+        otherwise every listed line ends with a spurious "^J". ``where`` was
+        unaffected because it strips the line before colorizing. See
+        gh-154470.
+        """
+        # colorize=True is what attaching from a color-capable terminal passes
+        # to the server, and it is what makes the server colorize ``list``.
+        script = textwrap.dedent(f"""
+            import pdb, sys
+            def helper():
+                x = 42
+                return x
+            def connect():
+                frame = sys._getframe()
+                pdb._connect(
+                    host='127.0.0.1',
+                    port={self.port},
+                    frame=frame,
+                    commands="",
+                    version=pdb._PdbServer.protocol_version(),
+                    signal_raising_thread=False,
+                    colorize=True,
+                )
+                return helper()
+            connect()
+        """)
+        self._create_script(script=script)
+        process, client_file = self._connect_and_get_client_file()
+
+        with kill_on_error(process):
+            self._read_until_prompt(client_file)
+            self._send_command(client_file, "l 1, 15")
+            messages = self._read_until_prompt(client_file)
+            source = "".join(m["message"] for m in messages if "message" in m)
+
+            # Sanity: we really did receive colorized source ...
+            self.assertIn("helper", source)
+            self.assertIn("\x1b[", source)  # ANSI color escapes are present
+            # ... and no trailing newline leaked through as caret notation.
+            self.assertNotIn("^J", source)
+            self._send_command(client_file, "c")
+            process.wait(timeout=SHORT_TIMEOUT)
+            self.assertEqual(process.returncode, 0)
+
     @unittest.skipUnless(pty, "requires pty")
     def test_prompt_with_interactive_terminal(self):
         """The server must send "(Pdb) " even when the target owns a terminal.

Misc/NEWS.d/next/Core_and_Builtins/2026-09-13-21-00-00.gh-issue-153568.prefix-cache.rst

@@ -0,0 +1,2 @@
+Speed up parsing by reusing memoized rule prefixes across consecutive grammar
+alternatives, avoiding repeated rule calls and cache lookups.

Misc/NEWS.d/next/Library/2026-07-22-13-39-58.gh-issue-154470.zGcz6K.rst

@@ -0,0 +1,4 @@
+Fixed a spurious ``^J`` printed at the end of every source line by the
+``list`` command of :mod:`pdb` when the output is colorized (for example
+when attaching to a running process). The source line is now stripped before
+it is colorized, like the ``where`` command already did.