Honor pyvenv.cfg in the launcher as per PEP405

Author: steve-s

graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java

@@ -25,10 +25,12 @@
  */
 package com.oracle.graal.python.shell;
 
+import java.io.BufferedReader;
 import java.io.EOFException;
 import java.io.File;
 import java.io.FileDescriptor;
 import java.io.FileOutputStream;
+import java.io.IOError;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -76,6 +78,8 @@ public static void main(String[] args) {
     // provided by GraalVM thin launcher
     protected static final String J_BASH_LAUNCHER_EXEC_PROPERTY_NAME = "org.graalvm.launcher.executablename";
 
+    private static final String J_PYENVCFG = "pyvenv.cfg";
+
     private static long startupWallClockTime = -1;
     private static long startupNanoTime = -1;
 
@@ -583,10 +587,21 @@ protected void launch(Builder contextBuilder) {
         if (executable != null) {
             contextBuilder.option("python.ExecutableList", executable);
         } else {
-            contextBuilder.option("python.Executable", getExecutable());
+            executable = getExecutable();
+            contextBuilder.option("python.Executable", executable);
             // The unlikely separator is used because options need to be
             // strings. See PythonOptions.getExecutableList()
             contextBuilder.option("python.ExecutableList", String.join("🏆", getExecutableList()));
+            // We try locating and loading options from pyvenv.cfg according to PEP405 as long as
+            // the user did not explicitly pass some options that would be otherwise loaded from
+            // pyvenv.cfg. Notable usage of this feature is GraalPython venvs which generate a
+            // launcher script that passes those options explicitly without relying on pyvenv.cfg
+            boolean tryVenvCfg = !hasContextOptionSetViaCommandLine("SysPrefix") &&
+                            !hasContextOptionSetViaCommandLine("PythonHome") &&
+                            System.getenv("GRAAL_PYTHONHOME") == null;
+            if (tryVenvCfg) {
+                findAndApplyVenvCfg(contextBuilder, executable);
+            }
         }
 
         // setting this to make sure our TopLevelExceptionHandler calls the excepthook
@@ -684,6 +699,39 @@ protected void launch(Builder contextBuilder) {
         System.exit(rc);
     }
 
+    private void findAndApplyVenvCfg(Builder contextBuilder, String executable) {
+        Path binDir = Paths.get(executable).getParent();
+        Path venvCfg = binDir.resolve(J_PYENVCFG);
+        if (!Files.exists(venvCfg)) {
+            venvCfg = binDir.getParent().resolve(J_PYENVCFG);
+            if (!Files.exists(venvCfg)) {
+                return; // not found
+            }
+        }
+        try (BufferedReader reader = Files.newBufferedReader(venvCfg)) {
+            String line;
+            while ((line = reader.readLine()) != null) {
+                String[] parts = line.split("=", 2);
+                if (parts.length != 2) {
+                    continue;
+                }
+                String name = parts[0].trim();
+                if (name.equals("home")) {
+                    contextBuilder.option("python.PythonHome", parts[1].trim());
+                    try {
+                        contextBuilder.option("python.SysPrefix", venvCfg.getParent().toAbsolutePath().toString());
+                    } catch (IOError ex) {
+                        System.err.println();
+                        throw abort("Could not set the home according to the pyvenv.cfg file.", 65);
+                    }
+                    break;
+                }
+            }
+        } catch (IOException ex) {
+            throw abort("Could not read the pyvenv.cfg file.", 66);
+        }
+    }
+
     private static boolean matchesPythonOption(String arg, String key) {
         assert !key.startsWith("python.");
         return arg.startsWith("--python." + key) || arg.startsWith("--" + key);
@@ -715,6 +763,18 @@ private String getContextOptionIfSetViaCommandLine(String key) {
         return null;
     }
 
+    private boolean hasContextOptionSetViaCommandLine(String key) {
+        if (System.getProperty("polyglot.python." + key) != null) {
+            return System.getProperty("polyglot.python." + key) != null;
+        }
+        for (String f : givenArguments) {
+            if (matchesPythonOption(f, key)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     private static void printFileNotFoundException(NoSuchFileException e) {
         String reason = e.getReason();
         if (reason == null) {

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java

@@ -499,6 +499,77 @@ public Object doIt(PFunction func,
         }
     }
 
+    @Builtin(name = "get_toolchain_tools_for_venv")
+    @TypeSystemReference(PythonArithmeticTypes.class)
+    @GenerateNodeFactory
+    public abstract static class GetToolchainToolsForVenv extends PythonBuiltinNode {
+        private static final class Tool {
+            final String name;
+            final boolean isVariableName;
+            final Object[] targets;
+
+            public Tool(String name, boolean isVariableName, Object[] targets) {
+                this.name = name;
+                this.isVariableName = isVariableName;
+                this.targets = targets;
+            }
+
+            static Tool forVariable(String name, Object... targets) {
+                return new Tool(name, true, targets);
+            }
+
+            static Tool forBinary(String name, Object... targets) {
+                return new Tool(name, true, targets);
+            }
+        }
+
+        static final Tool[] tools = new Tool[]{
+                        Tool.forVariable("AR", tsLiteral("ar")),
+                        Tool.forVariable("RANLIB", tsLiteral("ranlib")),
+                        Tool.forVariable("NM", tsLiteral("nm")),
+                        Tool.forVariable("LD", tsLiteral("ld.lld"), tsLiteral("ld"), tsLiteral("lld")),
+                        Tool.forVariable("CC", tsLiteral("clang"), tsLiteral("cc")),
+                        Tool.forVariable("CXX", tsLiteral("clang++"), tsLiteral("c++")),
+                        Tool.forBinary("llvm-as", tsLiteral("as")),
+                        Tool.forBinary("clang-cl", tsLiteral("cl")),
+                        Tool.forBinary("clang-cpp", tsLiteral("cpp")),
+        };
+
+        @Specialization
+        @TruffleBoundary
+        protected Object getToolPath() {
+            Env env = getContext().getEnv();
+            LanguageInfo llvmInfo = env.getInternalLanguages().get(J_LLVM_LANGUAGE);
+            Toolchain toolchain = env.lookup(llvmInfo, Toolchain.class);
+            List<TruffleFile> toolchainPaths = toolchain.getPaths("PATH");
+            EconomicMapStorage storage = EconomicMapStorage.create(tools.length);
+            for (Tool tool : tools) {
+                String path = null;
+                if (tool.isVariableName) {
+                    TruffleFile toolPath = toolchain.getToolPath(tool.name);
+                    if (toolPath != null) {
+                        path = toolPath.getAbsoluteFile().getPath();
+                    }
+                } else {
+                    for (TruffleFile toolchainPath : toolchainPaths) {
+                        LOGGER.finest(() -> " Testing path " + toolchainPath.getPath() + " for tool " + tool.name);
+                        TruffleFile pathToTest = toolchainPath.resolve(tool.name);
+                        if (pathToTest.exists()) {
+                            path = pathToTest.getAbsoluteFile().getPath();
+                            break;
+                        }
+                    }
+                }
+                if (path != null) {
+                    storage.putUncached(toTruffleStringUncached(path), factory().createTuple(tool.targets));
+                } else {
+                    LOGGER.info("Could not locate tool " + tool.name);
+                }
+            }
+            return factory().createDict(storage);
+        }
+    }
+
     /*
      * Internal check used in tests only to check that we are running through managed launcher.
      */

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java

@@ -1551,20 +1551,22 @@ public void initializeHomeAndPrefixPaths(Env newEnv, String languageHome) {
                         "\n\tCAPI: {5}" +
                         "\n\tJNI library: {6}", languageHome, sysPrefix, basePrefix, coreHome, stdLibHome, capiHome, jniHome));
 
-        String envHome = null;
-        try {
-            envHome = System.getenv("GRAAL_PYTHONHOME");
-        } catch (SecurityException e) {
+        String pythonHome = newEnv.getOptions().get(PythonOptions.PythonHome);
+        if (pythonHome.isEmpty()) {
+            try {
+                pythonHome = System.getenv("GRAAL_PYTHONHOME");
+            } catch (SecurityException e) {
+            }
         }
 
         final TruffleFile home;
-        if (languageHome != null && envHome == null) {
+        if (languageHome != null && pythonHome == null) {
             home = newEnv.getInternalTruffleFile(languageHome);
-        } else if (envHome != null) {
+        } else if (pythonHome != null) {
             boolean envHomeIsDirectory = false;
             TruffleFile envHomeFile = null;
             try {
-                envHomeFile = newEnv.getInternalTruffleFile(envHome);
+                envHomeFile = newEnv.getInternalTruffleFile(pythonHome);
                 envHomeIsDirectory = envHomeFile.isDirectory();
             } catch (SecurityException e) {
             }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonOptions.java

@@ -92,6 +92,10 @@ private PythonOptions() {
         // no instances
     }
 
+    @Option(category = OptionCategory.USER, help = "Set the home of Python. Equivalent of GRAAL_PYTHONHOME env variable. " +
+                    "Determines default values for the CoreHome, StdLibHome, SysBasePrefix, SysPrefix.", usageSyntax = "<path>", stability = OptionStability.STABLE) //
+    public static final OptionKey<String> PythonHome = new OptionKey<>("");
+
     @Option(category = OptionCategory.USER, help = "Set the location of sys.prefix. Overrides any environment variables or Java options.", usageSyntax = "<path>", stability = OptionStability.STABLE) //
     public static final OptionKey<TruffleString> SysPrefix = new OptionKey<>(T_EMPTY_STRING, TS_OPTION_TYPE);
 

graalpython/lib-python/3/venv/__init__.py

@@ -187,38 +187,12 @@ def create_if_needed(d):
 
     def _install_compilers(self, context):
         """Puts the Graal LLVM compiler tools on the path"""
-
-        # Table of well-known LLVM tools that must be queried by a variable name.
-        llvm_tools = {
-            "AR": ("ar",),
-            "RANLIB": ("ranlib",),
-            "NM": ("nm",),
-            "LD": ("ld.lld", "ld", "lld"),
-            "CC": ("clang", "cc"),
-            "CXX": ("clang++", "c++"),
-        }
-        # Table of additional LLVM tools to use if they are available.
-        _llvm_bins = {
-            "llvm-as": ("as",),
-            "clang-cl": ("cl",),
-            "clang-cpp": ("cpp",),
-        }
         bin_dir = os.path.join(context.env_dir, context.bin_name)
-        def create_symlinks(table, resolver):
-            for tool_var in table:
-                tool_path = resolver(tool_var)
-                if os.path.exists(tool_path):
-                    for name in table[tool_var]:
-                        dest = os.path.join(bin_dir, name)
-                        if not os.path.exists(dest):
-                            os.symlink(tool_path, dest)
-
-        create_symlinks(llvm_tools, __graalpython__.get_toolchain_tool_path)
-        # NOTE: function 'get_toolcahin_paths' returns a tuple
-        llvm_path = __graalpython__.get_toolchain_paths("PATH")
-        if llvm_path and llvm_path[0]:
-            create_symlinks(_llvm_bins, lambda binary_name: os.path.join(llvm_path[0], binary_name))
-
+        for (tool_path, names) in __graalpython__.get_toolchain_tools_for_venv().items():
+            for name in names:
+                dest = os.path.join(bin_dir, name)
+                if not os.path.exists(dest):
+                    os.symlink(tool_path, dest)
 
     def _patch_shebang(self, context):
         # Truffle change: we need to patch the pip/pip3 (and maybe other)