freeze site again, never return frozen modules when PYTHONPATH is given

Author: timfel

graalpython/com.oracle.graal.python.frozen/freeze_modules.py

@@ -71,9 +71,7 @@
             # We must explicitly mark os.path as a frozen module
             ("ntpath" if os.name == "nt" else "posixpath") + " : os.path",
             "os",
-            # TODO: freeze site when we upgrade to Python 3.9+ where the stdlib
-            # and setuptools can deal with a frozen site module
-            # "site",
+            "site",
             "stat",
         ],
     ),

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java

@@ -842,9 +842,12 @@ private void initializeJavaCore() {
         builtinsModule = builtinModules.get(BuiltinNames.BUILTINS);
     }
 
+    private static final String IMPORTLIB_NAME = "_frozen_importlib";
+    private static final String IMPORTLIB_EXTERNAL_NAME = "_frozen_importlib_external";
+
     private void initializeImportlib() {
-        String importlibName = "_frozen_importlib";
-        PythonModule bootstrap = ImpModuleBuiltins.importFrozenModuleObject(this, importlibName, false);
+        PythonModule bootstrap = ImpModuleBuiltins.importFrozenModuleObject(this, IMPORTLIB_NAME, false);
+        PythonModule bootstrapExternal;
 
         PyObjectCallMethodObjArgs callNode = PyObjectCallMethodObjArgs.getUncached();
         WriteAttributeToDynamicObjectNode writeNode = WriteAttributeToDynamicObjectNode.getUncached();
@@ -856,18 +859,21 @@ private void initializeImportlib() {
 
         if (bootstrap == null) {
             // true when the frozen module is not available
-            PythonModule bootstrapExternal = createModule("importlib._bootstrap_external");
-            writeNode.execute(bootstrapExternal, __PACKAGE__, "importlib");
-            setItem.execute(null, sysModules, "_frozen_importlib_external", bootstrapExternal);
+            bootstrapExternal = createModule("importlib._bootstrap_external");
             bootstrap = createModule("importlib._bootstrap");
-            writeNode.execute(bootstrap, __PACKAGE__, "importlib");
-            setItem.execute(null, sysModules, importlibName, bootstrap);
             loadFile("importlib/_bootstrap_external", getContext().getStdlibHome(), bootstrapExternal);
             loadFile("importlib/_bootstrap", getContext().getStdlibHome(), bootstrap);
         } else {
-            setItem.execute(null, sysModules, importlibName, bootstrap);
-            LOGGER.log(Level.FINE, () -> "import '_frozen_importlib' # <frozen>");
+            bootstrapExternal = ImpModuleBuiltins.importFrozenModuleObject(this, IMPORTLIB_EXTERNAL_NAME, true);
+            LOGGER.log(Level.FINE, () -> "import '" + IMPORTLIB_NAME + "' # <frozen>");
+            LOGGER.log(Level.FINE, () -> "import '" + IMPORTLIB_EXTERNAL_NAME + "' # <frozen>");
         }
+        setItem.execute(null, sysModules, IMPORTLIB_NAME, bootstrap);
+        setItem.execute(null, sysModules, IMPORTLIB_EXTERNAL_NAME, bootstrapExternal);
+
+        // __package__ needs to be set and doesn't get set by _bootstrap setup
+        writeNode.execute(bootstrap, __PACKAGE__, "importlib");
+        writeNode.execute(bootstrapExternal, __PACKAGE__, "importlib");
 
         callNode.execute(null, bootstrap, "_install", getSysModule(), lookupBuiltinModule("_imp"));
         writeNode.execute(getBuiltins(), "__import__", readNode.execute(bootstrap, "__import__"));
@@ -878,12 +884,6 @@ private void initializeImportlib() {
         PythonBuiltinClass moduleType = lookupType(PythonBuiltinClassType.PythonModule);
         writeNode.execute(moduleType, __REPR__, readNode.execute(bootstrap, "_module_repr"));
         SpecialMethodSlot.reinitializeSpecialMethodSlots(moduleType, getLanguage());
-
-        // __package__ needs to be set and doesn't get set by _bootstrap setup
-        writeNode.execute(bootstrap, __PACKAGE__, "importlib");
-
-        PythonModule bootstrapExternal = (PythonModule) getSysModules().getItem("_frozen_importlib_external");
-        writeNode.execute(bootstrapExternal, __PACKAGE__, "importlib");
     }
 
     private void initializePython3Core(String coreHome) {

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ImpModuleBuiltins.java

@@ -465,7 +465,9 @@ protected ArgumentClinicProvider getArgumentClinic() {
 
         @Specialization
         boolean run(String name) {
-            return findFrozen(name).status == FROZEN_OKAY;
+            // if PYTHONPATH is set, it is prepended to the sys.path on startup and thus might
+            // override any frozen modules from the stdlib
+            return getContext().getOption(PythonOptions.PythonPath).isEmpty() && findFrozen(name).status == FROZEN_OKAY;
         }
     }
 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/module/FrozenModules.java

@@ -195,6 +195,7 @@ private static final class Map {
         private static final PythonFrozenModule NTPATH = new PythonFrozenModule("NTPATH", "ntpath", false);
         private static final PythonFrozenModule POSIXPATH = new PythonFrozenModule("POSIXPATH", "posixpath", false);
         private static final PythonFrozenModule OS = new PythonFrozenModule("OS", "os", false);
+        private static final PythonFrozenModule SITE = new PythonFrozenModule("SITE", "site", false);
         private static final PythonFrozenModule STAT = new PythonFrozenModule("STAT", "stat", false);
         private static final PythonFrozenModule __HELLO__ = new PythonFrozenModule("__HELLO__", "__hello__", false);
         private static final PythonFrozenModule FROZEN_ONLY = new PythonFrozenModule("FROZEN_ONLY", "frozen_only", false);
@@ -527,6 +528,8 @@ public static final PythonFrozenModule lookup(String name) {
                 return Map.POSIXPATH;
             case "os":
                 return Map.OS;
+            case "site":
+                return Map.SITE;
             case "stat":
                 return Map.STAT;
             case "__hello__":