gh-60055: rebase onto main and document Request support

Author: orsenthil

Doc/library/urllib.robotparser.rst

@@ -21,14 +21,17 @@ website that published the :file:`robots.txt` file.  For more details on the
 structure of :file:`robots.txt` files, see :rfc:`9309`.
 
 
-.. class:: RobotFileParser(url='')
+.. class:: RobotFileParser(url_or_request='')
 
    This class provides methods to read, parse and answer questions about the
-   :file:`robots.txt` file at *url*.
+   :file:`robots.txt` file at *url* or a :class:`Request` object with additional
+   user-agent headers populated.
 
-   .. method:: set_url(url)
 
-      Sets the URL referring to a :file:`robots.txt` file.
+   .. method:: set_url(url_or_request)
+
+      Sets the URL referring to a :file:`robots.txt` file or a :class:`Request`
+      object with additional user-agent headers populated.
 
    .. method:: read()
 
@@ -102,3 +105,16 @@ class::
    True
    >>> rp.can_fetch("*", "http://www.pythontest.net/no-robots-here/")
    False
+
+
+The following example demonstrates use of a :class:`Request` object with additional user-agent headers populated::
+
+   >>> import urllib.robotparser
+   >>> import urllib.request
+   >>> rp = urllib.robotparser.RobotFileParser()
+   >>> rp.set_url(urllib.request.Request("http://en.wikipedia.org/robots.txt", headers={"User-Agent": "IsraBot"}))
+   >>> rp.read()
+   >>> rp.can_fetch("*", "https://en.wikipedia.org/")
+   True
+   >>> rp.can_fetch("*", "https://en.wikipedia.org/trap/")
+   False

Lib/urllib/robotparser.py

@@ -27,14 +27,14 @@ class RobotFileParser:
 
     """
 
-    def __init__(self, url=''):
+    def __init__(self, url_or_request=''):
         self.entries = []
         self.groups = {}
         self.sitemaps = []
         self.default_entry = None
         self.disallow_all = False
         self.allow_all = False
-        self.set_url(url)
+        self.set_url(url_or_request)
         self.last_checked = 0
 
     def mtime(self):
@@ -54,15 +54,17 @@ def modified(self):
         import time
         self.last_checked = time.time()
 
-    def set_url(self, url):
-        """Sets the URL referring to a robots.txt file."""
-        self.url = url
+    def set_url(self, url_or_request):
+        """Sets the URL referring to a robots.txt file. 
+        url_or_request can be a string or a Request object.
+        """
+        self.url = url_or_request
 
-        if isinstance(url, urllib.request.Request):
-            self.host = url.host
-            self.path = url.selector
+        if isinstance(url_or_request, urllib.request.Request):
+            self.host = url_or_request.host
+            self.path = url_or_request.selector
         else:
-            self.host, self.path = urllib.parse.urlparse(url)[1:3]
+            self.host, self.path = urllib.parse.urlparse(url_or_request)[1:3]
 
     def read(self):
         """Reads the robots.txt URL and feeds it to the parser."""