gh-60055: Allow passing a Request instance for RobotParser URLs (#103753)

* Allow passing a Request instance for the url parameter

* 📜🤖 Added by blurb_it.

* gh-60055: rebase onto main and document Request support

* Fix the lint warnings and CI Errors.

* Addressing RobotFileParser review comments.

* Update Doc/library/urllib.robotparser.rst

Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>

* Address additional Review Comments.

* Changed versionchanged to next.

---------

Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com>
Co-authored-by: Senthil Kumaran <senthil@python.org>
Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>

Author: 4 people

Doc/library/urllib.robotparser.rst

@@ -24,11 +24,18 @@ structure of :file:`robots.txt` files, see :rfc:`9309`.
 .. class:: RobotFileParser(url='')
 
    This class provides methods to read, parse and answer questions about the
-   :file:`robots.txt` file at *url*.
+   :file:`robots.txt` file at *url* or a :class:`urllib.request.Request` object.
+
+   .. versionchanged:: next
+     *url* parameter can be a :class:`urllib.request.Request` object.
 
    .. method:: set_url(url)
 
-      Sets the URL referring to a :file:`robots.txt` file.
+      Sets the URL referring to a :file:`robots.txt` file or a
+      :class:`urllib.request.Request` object.
+
+      .. versionchanged:: next
+        *url* parameter can be a :class:`urllib.request.Request` object.
 
    .. method:: read()
 
@@ -102,3 +109,17 @@ class::
    True
    >>> rp.can_fetch("*", "http://www.pythontest.net/no-robots-here/")
    False
+
+
+The following example demonstrates use of a :class:`urllib.request.Request`
+object with additional user-agent headers populated::
+
+   >>> import urllib.robotparser
+   >>> import urllib.request
+   >>> rp = urllib.robotparser.RobotFileParser()
+   >>> rp.set_url(urllib.request.Request("http://www.pythontest.net/robots.txt", headers={"User-Agent": "IsraBot"}))
+   >>> rp.read()
+   >>> rp.can_fetch("*", "http://www.pythontest.net/")
+   True
+   >>> rp.can_fetch("*", "http://www.pythontest.net/no-robots-here/")
+   False

Lib/test/test_robotparser.py

@@ -773,6 +773,37 @@ def testServiceUnavailable(self):
         self.assertFalse(parser.can_fetch("*", url + '/path/file.html'))
 
 
+class UserAgentSiteTestCase(BaseLocalNetworkTestCase, unittest.TestCase):
+
+    class RobotHandler(BaseHTTPRequestHandler):
+        def do_GET(self):
+            if self.headers.get('User-Agent').startswith('Python-urllib'):
+                self.send_error(403, "Forbidden access")
+            else:
+                self.send_response(200)
+                self.end_headers()
+                self.wfile.write(b"User-agent: *\nDisallow:")
+
+        def log_message(self, format, *args):
+            pass
+
+    def testUserAgentFilteringSite(self):
+        addr = self.server.server_address
+        url = f'http://{socket_helper.HOST}:{addr[1]}'
+        robots_url = url + "/robots.txt"
+        file_url = url + "/document"
+        parser = urllib.robotparser.RobotFileParser()
+        parser.set_url(robots_url)
+        parser.read()
+        self.assertTrue(parser.disallow_all)
+        self.assertFalse(parser.can_fetch("*", file_url))
+        parser = urllib.robotparser.RobotFileParser()
+        parser.set_url(urllib.request.Request(robots_url, headers={'User-Agent': 'cybermapper'}))
+        parser.read()
+        self.assertFalse(parser.disallow_all)
+        self.assertTrue(parser.can_fetch("*", file_url))
+
+
 @support.requires_working_socket()
 class NetworkTestCase(unittest.TestCase):
 

Lib/urllib/robotparser.py

@@ -55,8 +55,13 @@ def modified(self):
         self.last_checked = time.time()
 
     def set_url(self, url):
-        """Sets the URL referring to a robots.txt file."""
+        """Sets the URL referring to a robots.txt file.
+        can be a string or a Request object.
+        """
         self.url = url
+
+        if isinstance(url, urllib.request.Request):
+            url = url.full_url
         self.host, self.path = urllib.parse.urlsplit(url)[1:3]
 
     def read(self):

Misc/NEWS.d/next/Library/2023-04-24-11-12-00.gh-issue-60055.UjP6aX.rst

@@ -0,0 +1 @@
+Let ``urllib.robotparser.RobotFileParser`` accept a ``urllib.request.Request`` object as well as a url string when setting a robots.txt url.