gh-140950: Reset shlex state in push_source() so post-EOF pushes are read

After get_token() reaches end of input it sets self.state to None.
A subsequent push_source() did not reset state, so the next get_token()
call returned EOF immediately and the newly pushed source was silently
ignored.  pop_source() already resets state to ' ' for the same reason;
do the same in push_source().

Author: LeSingh1

Lib/shlex.py

@@ -81,6 +81,9 @@ def push_source(self, newstream, newfile=None):
         self.infile = newfile
         self.instream = newstream
         self.lineno = 1
+        # Reset state so that a source pushed after EOF (when state is None)
+        # is still read.  Mirrors pop_source().
+        self.state = ' '
         if self.debug:
             if newfile is not None:
                 print('shlex: pushing to file %s' % (self.infile,))

Lib/test/test_shlex.py

@@ -368,6 +368,18 @@ def testPunctuationCharsReadOnly(self):
         with self.assertRaises(AttributeError):
             shlex_instance.punctuation_chars = False
 
+    def testPushSourceAfterEOF(self):
+        # gh-140950: a source pushed after the current input has reached
+        # EOF must still be read.  Previously get_token() set state to None
+        # at EOF, and push_source() did not reset it, so the new source
+        # was silently ignored.
+        lexer = shlex.shlex('a')
+        self.assertEqual(lexer.get_token(), 'a')
+        self.assertEqual(lexer.get_token(), lexer.eof)
+        lexer.push_source('b')
+        self.assertEqual(lexer.get_token(), 'b')
+        self.assertEqual(lexer.get_token(), lexer.eof)
+
     @cpython_only
     def test_lazy_imports(self):
         import_helper.ensure_lazy_imports('shlex', {'collections', 're', 'os'})

Misc/NEWS.d/next/Library/2026-05-20-16-02-48.gh-issue-140950.3zwzaKTA.rst

@@ -0,0 +1,4 @@
+Fix :class:`shlex.shlex` ignoring a source pushed via
+:meth:`~shlex.shlex.push_source` after the current input had reached EOF.
+:meth:`!push_source` now resets the internal lexer state, mirroring
+:meth:`~shlex.shlex.pop_source`.