[GR-63253] GraalPy standalone fixes

PullRequest: graalpython/3791

Author: eregon

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java

@@ -32,15 +32,20 @@
 import static com.oracle.graal.python.nodes.BuiltinNames.T_NT;
 import static com.oracle.graal.python.nodes.BuiltinNames.T_STDERR;
 import static com.oracle.graal.python.nodes.BuiltinNames.T_SYS;
+import static com.oracle.graal.python.nodes.BuiltinNames.T_UNICODEDATA;
 import static com.oracle.graal.python.nodes.BuiltinNames.T_ZIPIMPORT;
+import static com.oracle.graal.python.nodes.BuiltinNames.T__SRE;
+import static com.oracle.graal.python.nodes.BuiltinNames.T__SYSCONFIG;
 import static com.oracle.graal.python.nodes.BuiltinNames.T__WEAKREF;
 import static com.oracle.graal.python.nodes.BuiltinNames.T___BUILTINS__;
+import static com.oracle.graal.python.nodes.BuiltinNames.T___GRAALPYTHON__;
 import static com.oracle.graal.python.nodes.BuiltinNames.T___IMPORT__;
 import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___DOC__;
 import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___PACKAGE__;
 import static com.oracle.graal.python.nodes.StringLiterals.J_PY_EXTENSION;
 import static com.oracle.graal.python.nodes.StringLiterals.T_DOT;
 import static com.oracle.graal.python.nodes.StringLiterals.T_GRAALPYTHON;
+import static com.oracle.graal.python.nodes.StringLiterals.T_JAVA;
 import static com.oracle.graal.python.nodes.StringLiterals.T_REF;
 import static com.oracle.graal.python.util.PythonUtils.toTruffleStringUncached;
 import static com.oracle.graal.python.util.PythonUtils.tsLiteral;
@@ -420,12 +425,12 @@ public abstract class Python3Core {
     private static TruffleString[] initializeCoreFiles() {
         // Order matters!
         List<TruffleString> coreFiles = List.of(
-                        toTruffleStringUncached("__graalpython__"),
-                        toTruffleStringUncached("_weakref"),
-                        toTruffleStringUncached("unicodedata"),
-                        toTruffleStringUncached("_sre"),
-                        toTruffleStringUncached("_sysconfig"),
-                        toTruffleStringUncached("java"),
+                        T___GRAALPYTHON__,
+                        T__WEAKREF,
+                        T_UNICODEDATA,
+                        T__SRE,
+                        T__SYSCONFIG,
+                        T_JAVA,
                         toTruffleStringUncached("pip_hook"));
         if (PythonOS.getPythonOS() == PythonOS.PLATFORM_WIN32) {
             coreFiles = new ArrayList<>(coreFiles);

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltins.java

@@ -36,6 +36,7 @@
 import java.util.Map.Entry;
 
 import com.oracle.graal.python.PythonLanguage;
+import com.oracle.graal.python.builtins.modules.ImpModuleBuiltins.ExecBuiltin;
 import com.oracle.graal.python.builtins.objects.PNone;
 import com.oracle.graal.python.builtins.objects.function.PBuiltinFunction;
 import com.oracle.graal.python.builtins.objects.module.PythonModule;
@@ -46,6 +47,7 @@
 import com.oracle.graal.python.runtime.object.PFactory;
 import com.oracle.graal.python.util.BiConsumer;
 import com.oracle.truffle.api.RootCallTarget;
+import com.oracle.truffle.api.TruffleLanguage.Env;
 import com.oracle.truffle.api.dsl.NodeFactory;
 import com.oracle.truffle.api.strings.TruffleString;
 
@@ -114,6 +116,11 @@ public void initialize(Python3Core core) {
     /**
      * Run any actions that can only be run in the post-initialization step, that is, if we're
      * actually going to start running rather than just pre-initializing.
+     * <p>
+     * See {@link Python3Core#postInitialize(Env)} as postInitialize() is only run under some
+     * conditions. See also {@link ExecBuiltin}, tough that does not get called if the built-in
+     * module is actually shadowed by a frozen one, or if the built-in module is actually added to
+     * sys.modules during context initialization before the importlib exists, etc.
      */
     public void postInitialize(@SuppressWarnings("unused") Python3Core core) {
         // nothing to do by default
@@ -156,10 +163,18 @@ public static int numDefaults(Builtin builtin) {
         return maxNumPosArgs - builtin.minNumOfPositionalArgs();
     }
 
+    /**
+     * May only be used in {@link #initialize} or before. Use {@link PythonObject#setAttribute}
+     * instead in {@link #postInitialize}.
+     */
     protected final void addBuiltinConstant(String name, Object value) {
         addBuiltinConstant(toTruffleStringUncached(name), value);
     }
 
+    /**
+     * May only be used in {@link #initialize} or before. Use {@link PythonObject#setAttribute}
+     * instead in {@link #postInitialize}.
+     */
     protected final void addBuiltinConstant(TruffleString name, Object value) {
         builtinConstants.put(name, ensureNoJavaString(value));
     }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/UnicodeDataModuleBuiltins.java

@@ -40,11 +40,15 @@
  */
 package com.oracle.graal.python.builtins.modules;
 
+import static com.oracle.graal.python.nodes.BuiltinNames.J_UNICODEDATA;
+import static com.oracle.graal.python.nodes.BuiltinNames.T_UNICODEDATA;
 import static com.oracle.graal.python.runtime.exception.PythonErrorType.ValueError;
 import static com.oracle.graal.python.util.PythonUtils.TS_ENCODING;
+import static com.oracle.graal.python.util.PythonUtils.toTruffleStringUncached;
 
 import java.util.List;
 
+import com.oracle.graal.python.builtins.objects.module.PythonModule;
 import org.graalvm.shadowed.com.ibm.icu.lang.UCharacter;
 import org.graalvm.shadowed.com.ibm.icu.lang.UProperty;
 import org.graalvm.shadowed.com.ibm.icu.text.Normalizer2;
@@ -73,14 +77,15 @@
 import com.oracle.truffle.api.nodes.Node;
 import com.oracle.truffle.api.strings.TruffleString;
 
-@CoreFunctions(defineModule = "unicodedata")
+@CoreFunctions(defineModule = J_UNICODEDATA, isEager = true)
 public final class UnicodeDataModuleBuiltins extends PythonBuiltins {
     @Override
     protected List<? extends NodeFactory<? extends PythonBuiltinBaseNode>> getNodeFactories() {
         return UnicodeDataModuleBuiltinsFactory.getFactories();
     }
 
-    public static String getUnicodeVersion() {
+    // Must not be used at image build time because ICU is --initialize-at-run-time
+    private static String getUnicodeVersion() {
         VersionInfo version = UCharacter.getUnicodeVersion();
         return Integer.toString(version.getMajor()) + '.' +
                         version.getMinor() + '.' +
@@ -104,9 +109,10 @@ private static String getUnicodeNameTB(int cp) {
     }
 
     @Override
-    public void initialize(Python3Core core) {
-        super.initialize(core);
-        addBuiltinConstant("unidata_version", getUnicodeVersion());
+    public void postInitialize(Python3Core core) {
+        super.postInitialize(core);
+        PythonModule self = core.lookupBuiltinModule(T_UNICODEDATA);
+        self.setAttribute(toTruffleStringUncached("unidata_version"), toTruffleStringUncached(getUnicodeVersion()));
     }
 
     static final int NORMALIZER_FORM_COUNT = 4;

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/BuiltinNames.java

@@ -480,4 +480,7 @@ public abstract class BuiltinNames {
 
     public static final String J_EXCEPTION_GROUP = "ExceptionGroup";
     public static final TruffleString T_EXCEPTION_GROUP = tsLiteral(J_EXCEPTION_GROUP);
+
+    public static final String J_UNICODEDATA = "unicodedata";
+    public static final TruffleString T_UNICODEDATA = tsLiteral(J_UNICODEDATA);
 }

mx.graalpython/mx_graalpython.py

@@ -60,7 +60,6 @@
 import mx_native
 import mx_unittest
 import mx_sdk
-import mx_sdk_vm_ng
 import mx_subst
 import mx_truffle
 import mx_graalpython_bisect
@@ -103,9 +102,6 @@ def get_boolean_env(name, default=False):
 LATEST_JAVA_HOME = {"JAVA_HOME": os.environ.get("LATEST_JAVA_HOME", mx.get_jdk().home)}
 RUNNING_ON_LATEST_JAVA = os.environ.get("LATEST_JAVA_HOME", os.environ.get("JAVA_HOME")) == mx.get_jdk().home
 
-if not SUITE._output_root_includes_config():
-    # Workaround bug in mx_sdk_vm_ng.JavaHomeDependency not being a project yet relying on the underlying get_output_root implementation
-    mx_sdk_vm_ng.JavaHomeDependency.get_output_root = lambda s: os.path.join(s.get_output_base(), s.name)
 
 # this environment variable is used by some of our maven projects and jbang integration to build against the unreleased master version during development
 os.environ["GRAALPY_VERSION"] = GRAAL_VERSION
@@ -647,8 +643,6 @@ def graalpy_standalone_home(standalone_type, enterprise=False, dev=False, build=
     else:
         env_file = 'native-ee' if enterprise else 'native-ce'
         standalone_dist = 'GRAALPY_NATIVE_STANDALONE'
-        if "GraalVM" in subprocess.check_output([get_jdk().java, '-version'], stderr=subprocess.STDOUT, universal_newlines=True):
-            assert False, "Cannot build a GraalPy native standalone with a Graal JDK, we only support latest for building the native images"
 
     mx_args = ['-p', SUITE.dir, *(['--env', env_file] if env_file else [])]
     mx_args.append("--extra-image-builder-argument=-g")

mx.graalpython/suite.py

@@ -838,7 +838,6 @@
                 "-Dorg.graalvm.launcher.class=com.oracle.graal.python.shell.GraalPythonMain",
                 # GraalPy standalone specific flags
                 "-J-Xms14g", # GR-46399: libpythonvm needs more than the default minimum of 8 GB to be built
-                "-H:-CopyLanguageResources",
                 "-Dpolyglot.python.PosixModuleBackend=native",
                 "-Dpolyglot.python.Sha3ModuleBackend=native",
                 "<bytecode_dsl_build_args>",