Merge branch 'main' into pyrepl-str

Author: edvilme

.github/workflows/reusable-san.yml

@@ -72,6 +72,8 @@ jobs:
     - name: MSan option setup
       if: inputs.sanitizer == 'MSan'
       run: |
+        sudo sysctl -w vm.mmap_rnd_bits=28  # Reduce ASLR to avoid MSan re-executing
+
         echo "MSAN_OPTIONS=${SAN_LOG_OPTION} allocator_may_return_null=1 handle_segv=0" >> "$GITHUB_ENV"
         # MSan reports false positives for memory initialized by libraries
         # that are not built with MSan, so disable modules that use them.

Include/internal/pycore_call.h

@@ -57,12 +57,6 @@ extern PyObject* _PyObject_Call(
     PyObject *args,
     PyObject *kwargs);
 
-extern PyObject * _PyObject_CallMethodFormat(
-    PyThreadState *tstate,
-    PyObject *callable,
-    const char *format,
-    ...);
-
 // Export for 'array' shared extension
 PyAPI_FUNC(PyObject*) _PyObject_CallMethod(
     PyObject *obj,

Include/internal/pycore_unicodeobject.h

@@ -468,7 +468,7 @@ extern void _PyUnicode_InternStatic(PyInterpreterState *interp, PyObject **);
 extern void _PyUnicode_ClearInterned(PyInterpreterState *interp);
 
 // Like PyUnicode_AsUTF8(), but check for embedded null characters.
-// Export for '_sqlite3' shared extension.
+// Export for '_sqlite3' shared extension, and for Argument Clinic code.
 PyAPI_FUNC(const char *) _PyUnicode_AsUTF8NoNUL(PyObject *);
 
 

Include/pymacro.h

@@ -199,19 +199,23 @@
         } while(0)
 #endif
 
-/* Get the number of elements in a visible array
-
-   This does not work on pointers, or arrays declared as [], or function
-   parameters. With correct compiler support, such usage will cause a build
-   error (see Py_BUILD_ASSERT_EXPR).
-
-   Written by Rusty Russell, public domain, http://ccodearchive.net/
-
-   Requires at GCC 3.1+ */
-#if (defined(__GNUC__) && !defined(__STRICT_ANSI__) && \
-    (((__GNUC__ == 3) && (__GNUC_MINOR__ >= 1)) || (__GNUC__ >= 4)))
-/* Two gcc extensions.
-   &a[0] degrades to a pointer: a different type from an array */
+// Get the number of elements in a visible array.
+//
+// This does not work on pointers, or arrays declared as [], or function
+// parameters. With correct compiler support, such usage will cause a build
+// error (see Py_BUILD_ASSERT_EXPR).
+//
+// Written by Rusty Russell, public domain, http://ccodearchive.net/
+//
+// Require GCC 4 (it works on GCC 3.1).
+//
+// Two GCC extensions: &a[0] degrades to a pointer, a different type from an
+// array.
+//
+// gh-158810: Do not use __builtin_types_compatible_p() in strict C ANSI mode
+// and on C++.
+#if (defined(__GNUC__) && __GNUC__ >= 4 \
+        && !defined(__STRICT_ANSI__) && !defined(__cplusplus))
 #define Py_ARRAY_LENGTH(array) \
     (sizeof(array) / sizeof((array)[0]) \
      + Py_BUILD_ASSERT_EXPR(!__builtin_types_compatible_p(typeof(array), \

InternalDocs/profiling_binary_format.md

@@ -47,8 +47,8 @@ The file consists of five required sections and one optional extension:
 |   String Table   |  Variable size
 +------------------+  frame_table_offset
 |   Frame Table    |  Variable size
-+------------------+  file_size - 64 (when stats are present)
-| Profile Stats    |  32 bytes (optional)
++------------------+  file_size - 88 (when stats are present)
+| Profile Stats    |  56 bytes (optional)
 +------------------+  file_size - 32
 |     Footer       |  32 bytes (fixed)
 +------------------+  file_size
@@ -220,6 +220,7 @@ The status byte is a bitfield encoding thread state at sample time:
 |  2  | THREAD_STATUS_UNKNOWN | Thread state could not be determined       |
 |  3  | THREAD_STATUS_GIL_REQUESTED | Thread is waiting to acquire the GIL  |
 |  4  | THREAD_STATUS_HAS_EXCEPTION | Thread has a pending exception         |
+|  5  | THREAD_STATUS_MAIN_THREAD | Thread is the interpreter's main thread |
 
 Multiple flags can be set simultaneously (e.g., a thread can hold the GIL
 while also running on CPU). Analysis tools use these to filter samples or

Lib/profiling/sampling/_sync_coordinator.py

@@ -168,6 +168,11 @@ def _execute_script(script_path: str, script_args: List[str], cwd: str) -> None:
     if not os.path.isfile(script_path):
         raise TargetError(f"Script not found: {script_path}")
 
+    script_dir = os.path.dirname(os.path.realpath(script_path))
+    if script_dir in sys.path:
+        sys.path.remove(script_dir)
+    sys.path.insert(0, script_dir)
+
     # Replace sys.argv to match original script call
     sys.argv = [script_path] + script_args
 

Lib/profiling/sampling/heatmap_collector.py

@@ -783,14 +783,14 @@ def _generate_file_html(self, output_path: Path, filename: str,
                           line_counts: Dict[int, int], self_counts: Dict[int, int],
                           file_stat: FileStats):
         """Generate HTML for a single source file with heatmap coloring."""
-        # Read source file
+        source_lines = [f"# Source file not available: {filename}"]
         try:
-            source_lines = Path(filename).read_text(encoding='utf-8', errors='replace').splitlines()
-        except (IOError, OSError) as e:
-            if not (filename.startswith('<') or filename.startswith('[') or
-                    filename in ('~', '...', '.') or len(filename) < 2):
-                print(f"Warning: Could not read source file {filename}: {e}")
-            source_lines = [f"# Source file not available: {filename}"]
+            path = Path(filename)
+            if path.is_file():
+                source_lines = path.read_text(
+                    encoding='utf-8', errors='replace').splitlines()
+        except (IOError, OSError):
+            pass
 
         # Generate HTML for each line
         max_samples = max(line_counts.values()) if line_counts else 1

Lib/profiling/sampling/stack_collector.py

@@ -68,6 +68,10 @@ def export(self, filename):
         return True
 
 
+# Allow for tree conversion and the dict/list frames in the Python JSON encoder.
+_FLAMEGRAPH_RECURSION_MARGIN = 6000
+
+
 class FlamegraphCollector(StackTraceCollector):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -166,34 +170,41 @@ def set_mode(self, mode):
         self.stats["mode"] = mode
 
     def export(self, filename):
-        flamegraph_data = self._convert_to_flamegraph_format()
-
-        # Debug output with string table statistics
-        num_functions = len(flamegraph_data.get("children", []))
-        total_time = flamegraph_data.get("value", 0)
-        string_count = len(self._string_table)
-        s1 = "" if num_functions == 1 else "s"
-        s2 = "" if total_time == 1 else "s"
-        s3 = "" if string_count == 1 else "s"
-        print(
-            f"Flamegraph data: {num_functions} root function{s1}, "
-            f"{total_time} total sample{s2}, "
-            f"{string_count} unique string{s3}"
-        )
-
-        if num_functions == 0:
+        # Converting the call tree recurses to the sampled stack depth.
+        old_limit = sys.getrecursionlimit()
+        sys.setrecursionlimit(old_limit + _FLAMEGRAPH_RECURSION_MARGIN)
+        try:
+            flamegraph_data = self._convert_to_flamegraph_format()
+
+            # Debug output with string table statistics
+            num_functions = len(flamegraph_data.get("children", []))
+            total_time = flamegraph_data.get("value", 0)
+            string_count = len(self._string_table)
+            s1 = "" if num_functions == 1 else "s"
+            s2 = "" if total_time == 1 else "s"
+            s3 = "" if string_count == 1 else "s"
             print(
-                "Warning: No functions found in profiling data. Check if sampling captured any data."
+                f"Flamegraph data: {num_functions} root function{s1}, "
+                f"{total_time} total sample{s2}, "
+                f"{string_count} unique string{s3}"
             )
-            return False
 
-        html_content = self._create_flamegraph_html(flamegraph_data)
+            if num_functions == 0:
+                print(
+                    "Warning: No functions found in profiling data. "
+                    "Check if sampling captured any data."
+                )
+                return False
 
-        with open(filename, "w", encoding="utf-8") as f:
-            f.write(html_content)
+            html_content = self._create_flamegraph_html(flamegraph_data)
 
-        print(f"Flamegraph saved to: {filename}")
-        return True
+            with open(filename, "w", encoding="utf-8") as f:
+                f.write(html_content)
+
+            print(f"Flamegraph saved to: {filename}")
+            return True
+        finally:
+            sys.setrecursionlimit(old_limit)
 
     @staticmethod
     @functools.lru_cache(maxsize=None)
@@ -487,7 +498,12 @@ def _get_source_lines(self, func):
             return None
 
     def _create_flamegraph_html(self, data):
-        data_json = json.dumps(data)
+        try:
+            data_json = json.dumps(data)
+        except RecursionError:
+            # The C encoder can exhaust the C stack independently of the
+            # Python recursion limit. iterencode() uses the Python encoder.
+            data_json = "".join(json.JSONEncoder().iterencode(data))
 
         template_dir = importlib.resources.files(__package__)
         vendor_dir = template_dir / "_vendor"
@@ -665,16 +681,16 @@ def _convert_to_flamegraph_format(self):
         current_stats = self._aggregate_path_samples(self._root)
         baseline_stats = self._aggregate_path_samples(self._baseline_collector._root)
 
-        # Scale baseline values to make them comparable, accounting for both
-        # sample count differences and sample interval differences.
+        # Express baseline samples in units of the current sample interval.
+        # Do not normalize by total profile duration: doing so makes unchanged
+        # functions appear different when another function becomes faster or
+        # slower.
         baseline_total = self._baseline_collector._total_samples
-        if baseline_total > 0 and self._total_samples > 0:
-            current_time = self._total_samples * self.sample_interval_usec
-            baseline_time = baseline_total * self._baseline_collector.sample_interval_usec
-            scale = current_time / baseline_time
-        elif baseline_total > 0:
-            # Current profile is empty - use interval-based scale for elided display
-            scale = self.sample_interval_usec / self._baseline_collector.sample_interval_usec
+        if baseline_total > 0:
+            scale = (
+                self._baseline_collector.sample_interval_usec
+                / self.sample_interval_usec
+            )
         else:
             scale = 1.0
 
@@ -891,6 +907,10 @@ def _add_elided_metadata(self, node, baseline_stats, scale, path):
         else:
             node["diff_pct"] = 0.0
 
+        # Scale geometry after computing metadata from raw baseline counts.
+        node["value"] = node.get("value", 0) * scale
+        node["self"] = node.get("self", 0) * scale
+
         if "children" in node and node["children"]:
             for child in node["children"]:
                 self._add_elided_metadata(child, baseline_stats, scale, current_path)

Lib/tempfile.py

@@ -119,11 +119,15 @@ def _sanitize_params(prefix, suffix, dir):
     output_type = _infer_return_type(prefix, suffix, dir)
     if suffix is None:
         suffix = output_type()
+    if _os.path.dirname(suffix):
+        raise ValueError("suffix can't contain a directory component")
     if prefix is None:
         if output_type is str:
             prefix = template
         else:
             prefix = _os.fsencode(template)
+    if _os.path.dirname(prefix):
+        raise ValueError("prefix can't contain a directory component")
     if dir is None:
         if output_type is str:
             dir = gettempdir()

Lib/test/test_cext/extension.c

@@ -83,6 +83,7 @@ test_macros(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args))
 {
     PyObject *obj, *dict;
     PyObject *slots[1];
+    int small_array[] = {2, 5, 7};
 
     // test Py_BUILD_ASSERT() and Py_BUILD_ASSERT_EXPR()
     Py_BUILD_ASSERT(sizeof(int) == sizeof(unsigned int));
@@ -134,6 +135,9 @@ test_macros(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args))
     Py_END_CRITICAL_SECTION();
     Py_DECREF(dict);
 
+    // Test Py_ARRAY_LENGTH()
+    assert(Py_ARRAY_LENGTH(small_array) == 3);
+
     Py_RETURN_NONE;
 }