[3.13] gh-153200: Fix math.isqrt() for int subclasses with overridden comparison operators (GH-153203) (GH-153225) (GH-153226)

The final check-and-correct comparison in the arbitrary precision path
could call a comparison operator overridden in an int subclass.
Compare by value with int's tp_richcompare.

(cherry picked from commit 3a1b547)
(cherry picked from commit a9a91d0)

Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

Author: 3 people

Lib/test/test_math.py

@@ -238,6 +238,19 @@ def __init__(self, value):
     def __index__(self):
         return self.value
 
+# int subclass with broken arithmetic operators; implementations must
+# convert their arguments to exact ints instead of using these.
+class BadIntSubclass(int):
+    def _binop(self, other='ignored', mod=None):
+        return 42
+    __add__ = __radd__ = __sub__ = __rsub__ = _binop
+    __mul__ = __rmul__ = __mod__ = __rmod__ = _binop
+    __divmod__ = __rdivmod__ = __pow__ = __rpow__ = _binop
+    __floordiv__ = __rfloordiv__ = _binop
+    __lshift__ = __rlshift__ = __rshift__ = __rrshift__ = _binop
+    __and__ = __rand__ = __or__ = __ror__ = __xor__ = __rxor__ = _binop
+    __lt__ = __le__ = __gt__ = __ge__ = _binop
+
 class BadDescr:
     def __get__(self, obj, objtype=None):
         raise ValueError
@@ -1112,6 +1125,16 @@ def __index__(self):
         self.assertIs(type(s), int)
         self.assertEqual(s, 41)
 
+        # Overridden operators of an int subclass must not affect the
+        # result.
+        s = math.isqrt(BadIntSubclass(10**20))
+        self.assertIs(type(s), int)
+        self.assertEqual(s, 10**10)
+
+        s = math.isqrt(BadIntSubclass(10**20 - 1))
+        self.assertIs(type(s), int)
+        self.assertEqual(s, 10**10 - 1)
+
         with self.assertRaises(ValueError):
             math.isqrt(IntegerLike(-3))
 

Misc/NEWS.d/next/Library/2026-07-06-12-00-00.gh-issue-153200.isqrtLt.rst

@@ -0,0 +1,3 @@
+Fix :func:`math.isqrt` returning an incorrect result for arguments not
+less than 2**64 that are instances of an :class:`int` subclass with an
+overridden comparison operator.

Modules/mathmodule.c

@@ -1794,16 +1794,20 @@ math_isqrt(PyObject *module, PyObject *n)
     /* The correct result is either a or a - 1. Figure out which, and
        decrement a if necessary. */
 
-    /* a_too_large = n < a * a */
+    /* a_too_large = n < a * a.  Compare by value: n can be an instance
+       of an int subclass with an overridden __lt__ method. */
     b = PyNumber_Multiply(a, a);
     if (b == NULL) {
         goto error;
     }
-    a_too_large = PyObject_RichCompareBool(n, b, Py_LT);
+    PyObject *cmp = PyLong_Type.tp_richcompare(n, b, Py_LT);
     Py_DECREF(b);
-    if (a_too_large == -1) {
+    if (cmp == NULL) {
         goto error;
     }
+    assert(PyBool_Check(cmp));
+    a_too_large = (cmp == Py_True);
+    Py_DECREF(cmp);
 
     if (a_too_large) {
         Py_SETREF(a, PyNumber_Subtract(a, _PyLong_GetOne()));