Addressing RobotFileParser review comments.

Author: orsenthil

Doc/library/urllib.robotparser.rst

@@ -21,18 +21,21 @@ website that published the :file:`robots.txt` file.  For more details on the
 structure of :file:`robots.txt` files, see :rfc:`9309`.
 
 
-.. class:: RobotFileParser(url_or_request='')
+.. class:: RobotFileParser(url='')
 
    This class provides methods to read, parse and answer questions about the
-   :file:`robots.txt` file at *url* or a :class:`urllib.request.Request` object
-   with additional user-agent headers populated.
+   :file:`robots.txt` file at *url* or a :class:`urllib.request.Request` object.
 
+   .. versionchanged:: 3.16
+     *url* parameter can be a :class:`urllib.request.Request` object.
 
-   .. method:: set_url(url_or_request)
+   .. method:: set_url(url)
 
       Sets the URL referring to a :file:`robots.txt` file or a
-      :class:`urllib.request.Request` object with additional user-agent headers
-      populated.
+      :class:`urllib.request.Request` object.
+
+      .. versionchanged:: 3.16
+        *url* parameter can be a :class:`urllib.request.Request` object.
 
    .. method:: read()
 
@@ -114,9 +117,9 @@ object with additional user-agent headers populated::
    >>> import urllib.robotparser
    >>> import urllib.request
    >>> rp = urllib.robotparser.RobotFileParser()
-   >>> rp.set_url(urllib.request.Request("http://en.wikipedia.org/robots.txt", headers={"User-Agent": "IsraBot"}))
+   >>> rp.set_url(urllib.request.Request("http://www.pythontest.net/robots.txt", headers={"User-Agent": "IsraBot"}))
    >>> rp.read()
-   >>> rp.can_fetch("*", "https://en.wikipedia.org/")
+   >>> rp.can_fetch("*", "http://www.pythontest.net/")
    True
-   >>> rp.can_fetch("*", "https://en.wikipedia.org/trap/")
+   >>> rp.can_fetch("*", "https://www.pythontest.net/no-robots-here/")
    False

Lib/test/test_robotparser.py

@@ -773,51 +773,23 @@ def testServiceUnavailable(self):
         self.assertFalse(parser.can_fetch("*", url + '/path/file.html'))
 
 
-class UserAgentRobotHandler(BaseHTTPRequestHandler):
+class UserAgentSiteTestCase(BaseLocalNetworkTestCase, unittest.TestCase):
 
-    def do_GET(self):
-        if self.headers.get('User-Agent').startswith('Python-urllib'):
-            self.send_error(403, "Forbidden access")
-        else:
-            self.send_response(200)
-            self.end_headers()
-            self.wfile.write(b"User-agent: *\nDisallow:")
-
-    def log_message(self, format, *args):
-        pass
-
-
-@unittest.skipUnless(
-    support.has_socket_support,
-    "Socket server requires working socket."
-)
-class UserAgentSiteTestCase(unittest.TestCase):
-
-    def setUp(self):
-        # clear _opener global variable
-        self.addCleanup(urllib.request.urlcleanup)
-
-        self.server = HTTPServer((socket_helper.HOST, 0), UserAgentRobotHandler)
-
-        self.t = threading.Thread(
-            name='HTTPServer serving',
-            target=self.server.serve_forever,
-            # Short poll interval to make the test finish quickly.
-            # Time between requests is short enough that we won't wake
-            # up spuriously too many times.
-            kwargs={'poll_interval':0.01})
-        self.t.daemon = True  # In case this function raises.
-        self.t.start()
+    class RobotHandler(BaseHTTPRequestHandler):
+        def do_GET(self):
+            if self.headers.get('User-Agent').startswith('Python-urllib'):
+                self.send_error(403, "Forbidden access")
+            else:
+                self.send_response(200)
+                self.end_headers()
+                self.wfile.write(b"User-agent: *\nDisallow:")
 
-    def tearDown(self):
-        self.server.shutdown()
-        self.t.join()
-        self.server.server_close()
+        def log_message(self, format, *args):
+            pass
 
-    @threading_helper.reap_threads
     def testUserAgentFilteringSite(self):
         addr = self.server.server_address
-        url = 'http://' + socket_helper.HOST + ':' + str(addr[1])
+        url = f'http://{socket_helper.HOST}:{addr[1]}'
         robots_url = url + "/robots.txt"
         file_url = url + "/document"
         parser = urllib.robotparser.RobotFileParser()

Lib/urllib/robotparser.py

@@ -27,14 +27,14 @@ class RobotFileParser:
 
     """
 
-    def __init__(self, url_or_request=''):
+    def __init__(self, url=''):
         self.entries = []
         self.groups = {}
         self.sitemaps = []
         self.default_entry = None
         self.disallow_all = False
         self.allow_all = False
-        self.set_url(url_or_request)
+        self.set_url(url)
         self.last_checked = 0
 
     def mtime(self):
@@ -54,17 +54,16 @@ def modified(self):
         import time
         self.last_checked = time.time()
 
-    def set_url(self, url_or_request):
+    def set_url(self, url):
         """Sets the URL referring to a robots.txt file.
-        url_or_request can be a string or a Request object.
+        can be a string or a Request object.
         """
-        self.url = url_or_request
+        self.url = url
 
-        if isinstance(url_or_request, urllib.request.Request):
-            self.host = url_or_request.host
-            self.path = url_or_request.selector
+        if isinstance(url, urllib.request.Request):
+            self.host, self.path = urllib.parse.urlsplit(url.full_url)[1:3]
         else:
-            self.host, self.path = urllib.parse.urlparse(url_or_request)[1:3]
+            self.host, self.path = urllib.parse.urlsplit(url)[1:3]
 
     def read(self):
         """Reads the robots.txt URL and feeds it to the parser."""

Misc/NEWS.d/next/Library/2023-04-24-11-12-00.gh-issue-60055.UjP6aX.rst

@@ -1 +1 @@
-Let ``urllib.robotparser.RobotParser`` accept a ``urllib.request.Request`` object as well as a url string when setting a robots.txt url.
+Let ``urllib.robotparser.RobotFileParser`` accept a ``urllib.request.Request`` object as well as a url string when setting a robots.txt url.