[GR-56099] Merge master in 24.1 release branch

PullRequest: graalpython/3426

Author: eregon

CHANGELOG.md

@@ -13,6 +13,7 @@ language runtime. The main focus is on user-observable behavior of the engine.
 * In Jython emulation mode we now magically fall back to calling Java getters or setters when using Python attribute access for non-visible properties. This can help migrating away from Jython if you relied on this behavior. 
 * The option `python.EmulateJython` to enable Jython emulation is now marked as stable, and can thus be relied upon in production.
 * Fixed parsing of pyvenv.cfg according to PEP 405, which is required to use [uv](https://github.com/astral-sh/uv?tab=readme-ov-file#uv) generated venvs with GraalPy.
+* Use https://www.graalvm.org/python/wheels/ as the default value for the `--extra-index-url` pip option. This will make it easy for users to install GraalPy binary wheels in the future.
 
 ## Version 24.0.0
 * We now provide a collection of recipes in the form of GitHub Actions to build popular native extensions on GraalPy. These provide a reproducible way for the community to build native extensions for GraalPy with the correct dependencies. See scripts/wheelbuilder/README.md for details.

ci.jsonnet

@@ -1 +1 @@
-{ "overlay": "9d0f6e553533cf857ff21d485094ece21f9e7771" }
+{ "overlay": "d304553512c0373cbe1fb5586546fc02068efd29" }

graalpython/com.oracle.graal.python.cext/modules/_sqlite/connection.c

@@ -199,10 +199,8 @@ pysqlite_connection_init_impl(pysqlite_Connection *self, PyObject *database,
     // Create and configure SQLite database object.
     sqlite3 *db;
     int rc;
-    // GraalPy change: workaround for GR-51314, revert to CPython original once fixed
-    char* cstr = PyBytes_AS_STRING(bytes);
     Py_BEGIN_ALLOW_THREADS
-    rc = sqlite3_open_v2(cstr, &db,
+    rc = sqlite3_open_v2(PyBytes_AS_STRING(bytes), &db,
                          SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE |
                          (uri ? SQLITE_OPEN_URI : 0), NULL);
     if (rc == SQLITE_OK) {

graalpython/com.oracle.graal.python.cext/src/dictobject.c

@@ -2074,7 +2074,6 @@ PyDict_Clear(PyObject *op)
     }
     ASSERT_CONSISTENT(mp);
 }
-#endif // GraalPy change
 
 /* Internal version of PyDict_Next that returns a hash value in addition
  * to the key and value.
@@ -2085,30 +2084,65 @@ int
 _PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey,
              PyObject **pvalue, Py_hash_t *phash)
 {
-    // GraalPy change: different implementation
-    PyObject *tresult = GraalPyTruffleDict_Next(op, *ppos);
-    if (tresult == NULL) {
-    	if(pkey != NULL) {
-    		*pkey = NULL;
-    	}
-    	if(pvalue != NULL) {
-    		*pvalue = NULL;
-    	}
-    	return 0;
-    }
-    if (pkey != NULL) {
-    	*pkey = PyTuple_GetItem(tresult, 0);
-    }
-    if (pvalue != NULL) {
-    	*pvalue = PyTuple_GetItem(tresult, 1);
-    }
-    if (phash != NULL) {
-    	*phash = PyLong_AsSsize_t(PyTuple_GetItem(tresult, 2));
-    }
-    *ppos = PyLong_AsSsize_t(PyTuple_GetItem(tresult, 3));
-    Py_DECREF(tresult);
+    Py_ssize_t i;
+    PyDictObject *mp;
+    PyObject *key, *value;
+    Py_hash_t hash;
+
+    if (!PyDict_Check(op))
+        return 0;
+    mp = (PyDictObject *)op;
+    i = *ppos;
+    if (mp->ma_values) {
+        assert(mp->ma_used <= SHARED_KEYS_MAX_SIZE);
+        if (i < 0 || i >= mp->ma_used)
+            return 0;
+        int index = get_index_from_order(mp, i);
+        value = mp->ma_values->values[index];
+
+        key = DK_UNICODE_ENTRIES(mp->ma_keys)[index].me_key;
+        hash = unicode_get_hash(key);
+        assert(value != NULL);
+    }
+    else {
+        Py_ssize_t n = mp->ma_keys->dk_nentries;
+        if (i < 0 || i >= n)
+            return 0;
+        if (DK_IS_UNICODE(mp->ma_keys)) {
+            PyDictUnicodeEntry *entry_ptr = &DK_UNICODE_ENTRIES(mp->ma_keys)[i];
+            while (i < n && entry_ptr->me_value == NULL) {
+                entry_ptr++;
+                i++;
+            }
+            if (i >= n)
+                return 0;
+            key = entry_ptr->me_key;
+            hash = unicode_get_hash(entry_ptr->me_key);
+            value = entry_ptr->me_value;
+        }
+        else {
+            PyDictKeyEntry *entry_ptr = &DK_ENTRIES(mp->ma_keys)[i];
+            while (i < n && entry_ptr->me_value == NULL) {
+                entry_ptr++;
+                i++;
+            }
+            if (i >= n)
+                return 0;
+            key = entry_ptr->me_key;
+            hash = entry_ptr->me_hash;
+            value = entry_ptr->me_value;
+        }
+    }
+    *ppos = i+1;
+    if (pkey)
+        *pkey = key;
+    if (pvalue)
+        *pvalue = value;
+    if (phash)
+        *phash = hash;
     return 1;
 }
+#endif // GraalPy change
 
 /*
  * Iterate over a dict.  Use like so:

graalpython/com.oracle.graal.python.processor/src/com/oracle/graal/python/processor/ArgumentClinicProcessor.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -90,11 +90,7 @@ public class ArgumentClinicProcessor extends AbstractProcessor {
 
     @Override
     public Set<String> getSupportedAnnotationTypes() {
-        HashSet<String> vals = new HashSet<>();
-        vals.add(ArgumentClinic.class.getName());
-        vals.add(ArgumentsClinic.class.getName());
-        vals.add(ClinicConverterFactory.class.getName());
-        return vals;
+        return Set.of(ArgumentClinic.class.getName(), ArgumentsClinic.class.getName(), ClinicConverterFactory.class.getName());
     }
 
     @Override
@@ -105,7 +101,7 @@ public SourceVersion getSupportedSourceVersion() {
     @Override
     public boolean process(Set<? extends TypeElement> annotations, RoundEnvironment roundEnv) {
         if (roundEnv.processingOver()) {
-            return false;
+            return true;
         }
         try {
             ConverterFactory.initBuiltins(processingEnv.getElementUtils());

graalpython/com.oracle.graal.python.processor/src/com/oracle/graal/python/processor/CApiBuiltinsProcessor.java

@@ -168,15 +168,17 @@ private static final class CApiBuiltinDesc {
         public final String name;
         public final VariableElement[] arguments;
         public final VariableElement returnType;
+        public final boolean acquireGil;
         public final String call;
         public final String factory;
         public int id;
 
-        public CApiBuiltinDesc(Element origin, String name, VariableElement returnType, VariableElement[] arguments, String call, String factory) {
+        public CApiBuiltinDesc(Element origin, String name, VariableElement returnType, VariableElement[] arguments, boolean acquireGil, String call, String factory) {
             this.origin = origin;
             this.name = name;
             this.returnType = returnType;
             this.arguments = arguments;
+            this.acquireGil = acquireGil;
             this.call = call;
             this.factory = factory;
         }
@@ -356,11 +358,12 @@ private void addCApiBuiltins(RoundEnvironment re, List<CApiBuiltinDesc> javaBuil
                         name = builtinName;
                     }
                     var ret = findValue(builtin, "ret", VariableElement.class);
+                    boolean acquireGil = findValue(builtin, "acquireGil", Boolean.class);
                     String call = name(findValue(builtin, "call", VariableElement.class));
                     // boolean inlined = findValue(builtin, "inlined", Boolean.class);
                     VariableElement[] args = findValues(builtin, "args", VariableElement.class).toArray(new VariableElement[0]);
                     if (((TypeElement) element).getQualifiedName().toString().equals("com.oracle.graal.python.builtins.objects.cext.capi.CApiFunction.Dummy")) {
-                        additionalBuiltins.add(new CApiBuiltinDesc(element, builtinName, ret, args, call, null));
+                        additionalBuiltins.add(new CApiBuiltinDesc(element, builtinName, ret, args, acquireGil, call, null));
                     } else {
                         if (!isValidReturnType(ret)) {
                             processingEnv.getMessager().printError(
@@ -387,7 +390,7 @@ private void addCApiBuiltins(RoundEnvironment re, List<CApiBuiltinDesc> javaBuil
                             genName += "NodeGen";
                         }
                         verifyNodeClass(((TypeElement) element), builtin);
-                        javaBuiltins.add(new CApiBuiltinDesc(element, name, ret, args, call, genName));
+                        javaBuiltins.add(new CApiBuiltinDesc(element, name, ret, args, acquireGil, call, genName));
                     }
                 }
             }
@@ -634,7 +637,7 @@ private void generateBuiltinRegistry(List<CApiBuiltinDesc> javaBuiltins) throws
         for (var builtin : javaBuiltins) {
             String argString = Arrays.stream(builtin.arguments).map(b -> "ArgDescriptor." + b).collect(Collectors.joining(", "));
             lines.add("    public static final CApiBuiltinExecutable " + builtin.name + " = new CApiBuiltinExecutable(\"" + builtin.name + "\", CApiCallPath." + builtin.call + ", ArgDescriptor." +
-                            builtin.returnType + ", new ArgDescriptor[]{" + argString + "}, " + builtin.id + ");");
+                            builtin.returnType + ", new ArgDescriptor[]{" + argString + "}, " + builtin.acquireGil + ", " + builtin.id + ");");
         }
         lines.add("");
         lines.add("    public static final CApiBuiltinExecutable[] builtins = {");
@@ -836,7 +839,7 @@ private void checkImports(List<CApiBuiltinDesc> builtins) throws IOException {
     @SuppressWarnings({"try", "unused"})
     public boolean process(Set<? extends TypeElement> annotations, RoundEnvironment re) {
         if (re.processingOver()) {
-            return false;
+            return true;
         }
 
         List<CApiBuiltinDesc> javaBuiltins = new ArrayList<>();

graalpython/com.oracle.graal.python.processor/src/com/oracle/graal/python/processor/GenerateEnumConstantsProcessor.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -74,7 +74,7 @@ public SourceVersion getSupportedSourceVersion() {
     @SuppressWarnings({"try", "unused"})
     public boolean process(Set<? extends TypeElement> annotations, RoundEnvironment re) {
         if (re.processingOver()) {
-            return false;
+            return true;
         }
         Set<? extends Element> annotatedElements = re.getElementsAnnotatedWith(GenerateEnumConstants.class);
         for (Element el : annotatedElements) {

graalpython/com.oracle.graal.python.processor/src/com/oracle/graal/python/processor/SlotsProcessor.java

@@ -87,7 +87,7 @@ public SourceVersion getSupportedSourceVersion() {
     @Override
     public boolean process(Set<? extends TypeElement> annotations, RoundEnvironment roundEnv) {
         if (roundEnv.processingOver()) {
-            return false;
+            return true;
         }
         try {
             doProcess(roundEnv);

graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java

@@ -30,7 +30,6 @@
 import java.io.File;
 import java.io.FileDescriptor;
 import java.io.FileOutputStream;
-import java.io.IOError;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -717,8 +716,7 @@ protected void launch(Builder contextBuilder) {
             // the user did not explicitly pass some options that would be otherwise loaded from
             // pyvenv.cfg. Notable usage of this feature is GraalPython venvs which generate a
             // launcher script that passes those options explicitly without relying on pyvenv.cfg
-            boolean tryVenvCfg = !hasContextOptionSetViaCommandLine("SysPrefix") &&
-                            !hasContextOptionSetViaCommandLine("PythonHome") &&
+            boolean tryVenvCfg = !hasContextOptionSetViaCommandLine("PythonHome") &&
                             getEnv("GRAAL_PYTHONHOME") == null;
             if (tryVenvCfg) {
                 findAndApplyVenvCfg(contextBuilder, executable);
@@ -872,14 +870,16 @@ private static String toAbsolutePath(String executable) {
         return Paths.get(executable).toAbsolutePath().toString();
     }
 
+    // Rough equivalent of CPython's pyvenv.cfg logic in Modules/getpath.py
     private void findAndApplyVenvCfg(Builder contextBuilder, String executable) {
-        Path binDir;
+        Path executablePath;
         try {
-            binDir = Paths.get(executable).getParent();
+            executablePath = Paths.get(executable);
         } catch (InvalidPathException e) {
-            log("cannot determine the parent directory of the executable");
+            log("cannot determine path of the executable");
             return;
         }
+        Path binDir = executablePath.getParent();
         if (binDir == null) {
             log("parent directory of the executable does not exist");
             return;
@@ -907,36 +907,52 @@ private void findAndApplyVenvCfg(Builder contextBuilder, String executable) {
                 }
                 String name = parts[0].trim();
                 if (name.equals("home")) {
-                    Path homeProperty = Paths.get(parts[1].trim());
-                    /*
-                     * (tfel): According to PEP 405, the home key is the directory of the Python
-                     * executable from which this virtual environment was created, that is, it
-                     * usually ends with "/bin" on a Unix system. On Windows, the base Python should
-                     * be in the top-level directory or under "\Scripts". To support running from
-                     * Maven artifacts where we don't have a working executable, we patched our
-                     * shipped venv module to set the home path without a "/bin" or "\\Scripts"
-                     * suffix, so we explicitly check for those two subfolder cases and otherwise
-                     * assume the home key is directly pointing to the Python home.
-                     */
-                    if (homeProperty.endsWith("bin") || homeProperty.endsWith("Scripts")) {
-                        homeProperty = homeProperty.getParent();
-                    }
                     try {
-                        contextBuilder.option("python.PythonHome", homeProperty.toString());
-                    } catch (NullPointerException ex) {
+                        Path homeProperty = Paths.get(parts[1].trim());
+                        Path graalpyHome = homeProperty;
+                        /*
+                         * (tfel): According to PEP 405, the home key is the directory of the Python
+                         * executable from which this virtual environment was created, that is, it
+                         * usually ends with "/bin" on a Unix system. On Windows, the base Python
+                         * should be in the top-level directory or under "\Scripts". To support
+                         * running from Maven artifacts where we don't have a working executable, we
+                         * patched our shipped venv module to set the home path without a "/bin" or
+                         * "\\Scripts" suffix, so we explicitly check for those two subfolder cases
+                         * and otherwise assume the home key is directly pointing to the Python
+                         * home.
+                         */
+                        if (graalpyHome.endsWith("bin") || graalpyHome.endsWith("Scripts")) {
+                            graalpyHome = graalpyHome.getParent();
+                        }
+                        contextBuilder.option("python.PythonHome", graalpyHome.toString());
+                        /*
+                         * First try to resolve symlinked executables, since that may be more
+                         * accurate than assuming the executable in 'home'.
+                         */
+                        Path baseExecutable = null;
+                        try {
+                            Path realPath = executablePath.toRealPath();
+                            if (!realPath.equals(executablePath.toAbsolutePath())) {
+                                baseExecutable = realPath;
+                            }
+                        } catch (IOException ex) {
+                            // Ignore
+                        }
+                        if (baseExecutable == null) {
+                            baseExecutable = homeProperty.resolve(executablePath.getFileName());
+                        }
+                        if (Files.exists(baseExecutable)) {
+                            contextBuilder.option("python.BaseExecutable", baseExecutable.toString());
+                            /*
+                             * This is needed to support the legacy GraalVM layout where the
+                             * executable is a symlink into the 'languages' directory.
+                             */
+                            contextBuilder.option("python.PythonHome", baseExecutable.getParent().getParent().toString());
+                        }
+                    } catch (NullPointerException | InvalidPathException ex) {
                         // NullPointerException covers the possible null result of getParent()
                         warn("Could not set PYTHONHOME according to the pyvenv.cfg file.");
                     }
-                    String sysPrefix = null;
-                    try {
-                        sysPrefix = venvCfg.getParent().toAbsolutePath().toString();
-                    } catch (IOError | NullPointerException ex) {
-                        // NullPointerException covers the possible null result of getParent()
-                        warn("Could not set the sys.prefix according to the pyvenv.cfg file.");
-                    }
-                    if (sysPrefix != null) {
-                        contextBuilder.option("python.SysPrefix", sysPrefix);
-                    }
                     break;
                 }
             }

graalpython/com.oracle.graal.python.test.integration/src/com/oracle/graal/python/test/integration/PythonTests.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2023, Oracle and/or its affiliates.
+ * Copyright (c) 2017, 2024, Oracle and/or its affiliates.
  * Copyright (c) 2013, Regents of the University of California
  *
  * All rights reserved.
@@ -44,7 +44,7 @@
 import org.graalvm.polyglot.PolyglotException;
 import org.graalvm.polyglot.Source;
 import org.graalvm.polyglot.Value;
-import org.junit.Assert;
+import org.hamcrest.MatcherAssert;
 
 import com.oracle.graal.python.test.integration.advanced.BenchmarkTests;
 
@@ -106,7 +106,7 @@ public static void assertLastLineErrorContains(String expected, String code) {
         PythonTests.runThrowableScript(new String[0], source, System.out, printStream);
         String[] output = byteArray.toString().split("\n");
         // ignore the traceback
-        Assert.assertThat(output[output.length - 1], containsString(expected.trim()));
+        MatcherAssert.assertThat(output[output.length - 1], containsString(expected.trim()));
     }
 
     public static void assertPrintContains(String expected, String code) {