[GR-40690] Bytecode instrumentation

PullRequest: graalpython/2426

Author: msimacek

CHANGELOG.md

@@ -5,7 +5,7 @@ language runtime. The main focus is on user-observable behavior of the engine.
 
 ## Version 22.3.0
 * Rename GraalPython to GraalPy. This change also updates the launchers we ship to include symlinks from `python` and `python3` to `graalpy`, for better integration with other tools.
-* New interpreter backend based on interpreting bytecode. This change should bring better startup performance and memory footprint while retaining good JIT-compiled performance. There is no support for GraalVM instrumentation tools on the bytecode backend yet, so using one of the instrumentation options (e.g. `--inspect`) falls back on the AST backend.
+* New interpreter backend based on interpreting bytecode. This change should bring better startup performance and memory footprint while retaining good JIT-compiled performance.
 * New parser generated from CPython's new PEG grammar definition. It brings better compatibility and enables us to implement the `ast` module.
 * Added support for tracing API (`sys.settrace`) which makes `pdb` and related tools work on GraalPy.
 * Updated our pip support to automatically choose the best version for known packages. You can use `pip install pandas`, and pip will select the versions of pandas and numpy that we test in the GraalPy CI.

graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java

@@ -104,8 +104,6 @@ public static void main(String[] args) {
 
     boolean useASTInterpreter = false;
 
-    private String toolInstrumentWarning = null;
-
     protected static void setStartupTime() {
         if (GraalPythonMain.startupNanoTime == -1) {
             GraalPythonMain.startupNanoTime = System.nanoTime();
@@ -201,11 +199,7 @@ protected List<String> preprocessArguments(List<String> givenArgs, Map<String, S
                             useASTInterpreter = true;
                             continue;
                         default:
-                            if (argStartsWith(arg, "--agentscript", "--coverage", "--cpusampler", "--cputracer", "--dap",
-                                            "--heap.", "--heapmonitor", "--insight", "--inspect", "--lsp", "--memtracer", "--sandbox.")) {
-                                useASTInterpreter = true;
-                                toolInstrumentWarning = "WARNING: Switching to AST interpreter due to instruments option " + arg;
-                            } else if (arg.startsWith("--llvm.") ||
+                            if (arg.startsWith("--llvm.") ||
                                             matchesPythonOption(arg, "CoreHome") ||
                                             matchesPythonOption(arg, "StdLibHome") ||
                                             matchesPythonOption(arg, "CAPI") ||
@@ -597,6 +591,7 @@ protected void launch(Builder contextBuilder) {
         contextBuilder.option("python.VerboseFlag", Boolean.toString(verboseFlag));
         contextBuilder.option("python.IsolateFlag", Boolean.toString(isolateFlag));
         contextBuilder.option("python.WarnOptions", warnOptions);
+        contextBuilder.option("python.DontWriteBytecodeFlag", Boolean.toString(dontWriteBytecode));
         if (verboseFlag) {
             contextBuilder.option("log.python.level", "FINE");
         }
@@ -627,10 +622,6 @@ protected void launch(Builder contextBuilder) {
             contextBuilder.option("python.PyCachePrefix", "/dev/null");
             contextBuilder.option("python.EnableBytecodeInterpreter", "false");
             contextBuilder.option("python.DisableFrozenModules", "true");
-            if (toolInstrumentWarning != null) {
-                System.out.println(toolInstrumentWarning);
-                toolInstrumentWarning = null;
-            }
         } else {
             contextBuilder.option("python.DontWriteBytecodeFlag", Boolean.toString(dontWriteBytecode));
             if (cachePrefix != null) {
@@ -690,15 +681,6 @@ private static boolean matchesPythonOption(String arg, String key) {
         return arg.startsWith("--python." + key) || arg.startsWith("--" + key);
     }
 
-    private static boolean argStartsWith(String arg, String... tools) {
-        for (String t : tools) {
-            if (arg.startsWith(t)) {
-                return true;
-            }
-        }
-        return false;
-    }
-
     private String getContextOptionIfSetViaCommandLine(String key) {
         if (System.getProperty("polyglot.python." + key) != null) {
             return System.getProperty("polyglot.python." + key);

graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/compiler/SourceMapTests.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package com.oracle.graal.python.test.compiler;
+
+import java.util.Arrays;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.oracle.graal.python.compiler.OpCodes;
+import com.oracle.graal.python.compiler.SourceMap;
+
+public class SourceMapTests {
+    @Test
+    public void testRoundTrip() {
+        int[] startLines = new int[]{50, 50, 51, 51, 56, 256};
+        int[] startColumns = new int[]{0, 128, 0, 129, 5, 597};
+        int[] endLines = new int[]{50, 51, 51, 53, 59, 350};
+        int[] endColumns = new int[]{10, 129, 1024, 130, 134, 598};
+        int startLine = 49;
+        int startColumn = 0;
+        SourceMap.Builder builder = new SourceMap.Builder(startLine, startColumn);
+        byte[] code = new byte[6];
+        Arrays.fill(code, (byte) OpCodes.NOP.ordinal());
+        for (int i = 0; i < code.length; i++) {
+            builder.appendLocation(startLines[i], startColumns[i], endLines[i], endColumns[i]);
+        }
+        byte[] table = builder.build();
+        SourceMap map = new SourceMap(code, table, startLine, startColumn);
+        Assert.assertArrayEquals(startLines, map.startLineMap);
+        Assert.assertArrayEquals(startColumns, map.startColumnMap);
+        Assert.assertArrayEquals(endLines, map.endLineMap);
+        Assert.assertArrayEquals(endColumns, map.endColumnMap);
+    }
+}

graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/debug/PythonDebugTest.java

@@ -83,7 +83,6 @@ public void before() {
         newBuilder.allowAllAccess(true);
         PythonTests.closeContext();
         tester = new DebuggerTester(newBuilder);
-        PythonTests.skipOnBytecodeInterpreter();
     }
 
     @After
@@ -183,6 +182,29 @@ public void testSteppingAsExpected() throws Throwable {
         }
     }
 
+    @Test
+    public void testException() throws Throwable {
+        final Source source = Source.newBuilder("python", "" +
+                        "try:\n" +
+                        "  1 / 0\n" +
+                        "except BaseException as e:\n" +
+                        "  str(e)\n" +
+                        "1", "test_exception.py").buildLiteral();
+
+        try (DebuggerSession session = tester.startSession()) {
+            session.install(Breakpoint.newExceptionBuilder(true, true).build());
+            tester.startEval(source);
+
+            expectSuspended((SuspendedEvent event) -> {
+                DebugStackFrame frame = event.getTopStackFrame();
+                assertEquals(2, frame.getSourceSection().getStartLine());
+                assertNotNull(event.getException());
+                event.prepareContinue();
+            });
+            assertEquals("1", tester.expectDone());
+        }
+    }
+
     @Test
     public void testInlineEvaluation() throws Throwable {
         final Source source = Source.newBuilder("python", "" +
@@ -216,6 +238,28 @@ public void testInlineEvaluation() throws Throwable {
         }
     }
 
+    @Test
+    @SuppressWarnings("try")
+    public void testBreakpointBuiltin() throws Throwable {
+        final Source source = Source.newBuilder("python", "" +
+                        "def foo():\n" +
+                        "  a = 1\n" +
+                        "  breakpoint()\n" +
+                        "  return 1\n" +
+                        "foo()\n", "test_breakpoint_builtin.py").buildLiteral();
+
+        try (DebuggerSession session = tester.startSession()) {
+            tester.startEval(source);
+
+            expectSuspended((SuspendedEvent event) -> {
+                DebugStackFrame frame = event.getTopStackFrame();
+                assertEquals(3, frame.getSourceSection().getStartLine());
+                event.prepareContinue();
+            });
+            assertEquals("1", tester.expectDone());
+        }
+    }
+
     @Test
     public void testConditionalBreakpointInFunction() throws Throwable {
         final Source source = Source.newBuilder("python", "" +

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_multiprocessing_main_handling.txt

@@ -11,12 +11,3 @@
 *graalpython.lib-python.3.test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_script_compiled
 *graalpython.lib-python.3.test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_zipfile
 *graalpython.lib-python.3.test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_zipfile_compiled
-*graalpython.lib-python.3.test.test_multiprocessing_main_handling.SpawnCmdLineTest.test_basic_script
-*graalpython.lib-python.3.test.test_multiprocessing_main_handling.SpawnCmdLineTest.test_basic_script_no_suffix
-*graalpython.lib-python.3.test.test_multiprocessing_main_handling.SpawnCmdLineTest.test_directory
-*graalpython.lib-python.3.test.test_multiprocessing_main_handling.SpawnCmdLineTest.test_directory_compiled
-*graalpython.lib-python.3.test.test_multiprocessing_main_handling.SpawnCmdLineTest.test_ipython_workaround
-*graalpython.lib-python.3.test.test_multiprocessing_main_handling.SpawnCmdLineTest.test_module_in_package_in_zipfile
-*graalpython.lib-python.3.test.test_multiprocessing_main_handling.SpawnCmdLineTest.test_package
-*graalpython.lib-python.3.test.test_multiprocessing_main_handling.SpawnCmdLineTest.test_package_compiled
-*graalpython.lib-python.3.test.test_multiprocessing_main_handling.SpawnCmdLineTest.test_zipfile

graalpython/com.oracle.graal.python.test/testData/goldenFiles/CompilerTests/testAnnAssignment.co

@@ -1,10 +1,10 @@
 Disassembly of <module>:
-000000         0 SETUP_ANNOTATIONS
-000009         1 LOAD_BYTE_O                       12
-000000         3 STORE_NAME                        0    (a)
-000003         5 LOAD_NAME                         1    (int)
-000000         7 LOAD_NAME                         2    (__annotations__)
-000000         9 LOAD_STRING                       0    ('a')
-000000        11 STORE_SUBSCR
-000000        12 LOAD_NONE
-000000        13 RETURN_VALUE
+  1:0   -   1:11        0 SETUP_ANNOTATIONS
+  1:9   -   1:11        1 LOAD_BYTE_O                       12
+  1:0   -   1:1         3 STORE_NAME                        0    (a)
+  1:3   -   1:6         5 LOAD_NAME                         1    (int)
+  1:0   -   1:11        7 LOAD_NAME                         2    (__annotations__)
+  1:0   -   1:11        9 LOAD_STRING                       0    ('a')
+  1:0   -   1:11       11 STORE_SUBSCR
+  1:0   -   1:11       12 LOAD_NONE
+  1:0   -   1:11       13 RETURN_VALUE

graalpython/com.oracle.graal.python.test/testData/goldenFiles/CompilerTests/testArgAnnotations.co

@@ -1,30 +1,30 @@
 Disassembly of <module>:
-000000         0 COLLECTION_FROM_STACK             0    (PKeyword[])
-000000         2 LOAD_STRING                       0    ('b')
-000018         4 LOAD_BYTE_O                       2
-000000         6 LOAD_STRING                       1    ('a')
-000010         8 LOAD_BYTE_O                       1
-000000        10 LOAD_STRING                       2    ('c')
-000024        12 LOAD_BYTE_O                       3
-000000        14 LOAD_STRING                       3    ('d')
-000029        16 LOAD_BYTE_O                       4
-000000        18 LOAD_STRING                       4    ('e')
-000036        20 LOAD_BYTE_O                       5
-000000        22 COLLECTION_FROM_STACK             10   (dict)
-000000        24 MAKE_FUNCTION                     6    (foo)
-000000        27 STORE_NAME                        0    (foo)
-000000        29 LOAD_NONE
-000000        30 RETURN_VALUE
+  1:0   -   2:22        0 COLLECTION_FROM_STACK             0    (PKeyword[])
+  1:0   -   2:22        2 LOAD_STRING                       0    ('b')
+  1:18  -   1:19        4 LOAD_BYTE_O                       2
+  1:0   -   2:22        6 LOAD_STRING                       1    ('a')
+  1:10  -   1:11        8 LOAD_BYTE_O                       1
+  1:0   -   2:22       10 LOAD_STRING                       2    ('c')
+  1:24  -   1:25       12 LOAD_BYTE_O                       3
+  1:0   -   2:22       14 LOAD_STRING                       3    ('d')
+  1:29  -   1:30       16 LOAD_BYTE_O                       4
+  1:0   -   2:22       18 LOAD_STRING                       4    ('e')
+  1:36  -   1:37       20 LOAD_BYTE_O                       5
+  1:0   -   2:22       22 COLLECTION_FROM_STACK             10   (dict)
+  1:0   -   2:22       24 MAKE_FUNCTION                     6    (foo)
+  1:0   -   2:22       27 STORE_NAME                        0    (foo)
+  1:0   -   2:22       29 LOAD_NONE
+  1:0   -   2:22       30 RETURN_VALUE
 
 Disassembly of foo:
-000042         0 LOAD_GLOBAL                       0    (print)
-000048         2 LOAD_FAST                         0    (a)
-000051         4 LOAD_FAST                         1    (b)
-000054         6 LOAD_FAST                         3    (c)
-000057         8 LOAD_FAST                         2    (d)
-000060        10 LOAD_FAST                         4    (e)
-000042        12 COLLECTION_FROM_STACK             5    (Object[])
-000042        14 CALL_FUNCTION_VARARGS
-000042        15 POP_TOP
-000042        16 LOAD_NONE
-000042        17 RETURN_VALUE
+  2:2   -   2:7         0 LOAD_GLOBAL                       0    (print)
+  2:8   -   2:9         2 LOAD_FAST                         0    (a)
+  2:11  -   2:12        4 LOAD_FAST                         1    (b)
+  2:14  -   2:15        6 LOAD_FAST                         3    (c)
+  2:17  -   2:18        8 LOAD_FAST                         2    (d)
+  2:20  -   2:21       10 LOAD_FAST                         4    (e)
+  2:2   -   2:22       12 COLLECTION_FROM_STACK             5    (Object[])
+  2:2   -   2:22       14 CALL_FUNCTION_VARARGS
+  2:2   -   2:22       15 POP_TOP
+  2:2   -   2:22       16 LOAD_NONE
+  2:2   -   2:22       17 RETURN_VALUE

graalpython/com.oracle.graal.python.test/testData/goldenFiles/CompilerTests/testArgsCombination.co

@@ -1,19 +1,19 @@
 Disassembly of <module>:
-000000         0 COLLECTION_FROM_STACK             0    (PKeyword[])
-000000         2 MAKE_FUNCTION                     2    (foo)
-000000         5 STORE_NAME                        0    (foo)
-000000         7 LOAD_NONE
-000000         8 RETURN_VALUE
+  1:0   -   2:22        0 COLLECTION_FROM_STACK             0    (PKeyword[])
+  1:0   -   2:22        2 MAKE_FUNCTION                     2    (foo)
+  1:0   -   2:22        5 STORE_NAME                        0    (foo)
+  1:0   -   2:22        7 LOAD_NONE
+  1:0   -   2:22        8 RETURN_VALUE
 
 Disassembly of foo:
-000032         0 LOAD_GLOBAL                       0    (print)
-000038         2 LOAD_FAST                         0    (a)
-000041         4 LOAD_FAST                         1    (b)
-000044         6 LOAD_FAST                         3    (c)
-000047         8 LOAD_FAST                         2    (d)
-000050        10 LOAD_FAST                         4    (e)
-000032        12 COLLECTION_FROM_STACK             5    (Object[])
-000032        14 CALL_FUNCTION_VARARGS
-000032        15 POP_TOP
-000032        16 LOAD_NONE
-000032        17 RETURN_VALUE
+  2:2   -   2:7         0 LOAD_GLOBAL                       0    (print)
+  2:8   -   2:9         2 LOAD_FAST                         0    (a)
+  2:11  -   2:12        4 LOAD_FAST                         1    (b)
+  2:14  -   2:15        6 LOAD_FAST                         3    (c)
+  2:17  -   2:18        8 LOAD_FAST                         2    (d)
+  2:20  -   2:21       10 LOAD_FAST                         4    (e)
+  2:2   -   2:22       12 COLLECTION_FROM_STACK             5    (Object[])
+  2:2   -   2:22       14 CALL_FUNCTION_VARARGS
+  2:2   -   2:22       15 POP_TOP
+  2:2   -   2:22       16 LOAD_NONE
+  2:2   -   2:22       17 RETURN_VALUE

graalpython/com.oracle.graal.python.test/testData/goldenFiles/CompilerTests/testAssignment.co

@@ -1,5 +1,5 @@
 Disassembly of <module>:
-000004         0 LOAD_BYTE_O                       12
-000000         2 STORE_NAME                        0    (a)
-000000         4 LOAD_NONE
-000000         5 RETURN_VALUE
+  1:4   -   1:6         0 LOAD_BYTE_O                       12
+  1:0   -   1:1         2 STORE_NAME                        0    (a)
+  1:0   -   1:6         4 LOAD_NONE
+  1:0   -   1:6         5 RETURN_VALUE

graalpython/com.oracle.graal.python.test/testData/goldenFiles/CompilerTests/testAugAssignment.co

@@ -1,7 +1,7 @@
 Disassembly of <module>:
-000000         0 LOAD_NAME                         0    (a)
-000005         2 LOAD_DOUBLE_O                     0    (12.0)
-000000         4 BINARY_OP                         1    (INPLACE_ADD)
-000000         6 STORE_NAME                        0    (a)
-000000         8 LOAD_NONE
-000000         9 RETURN_VALUE
+  1:0   -   1:1         0 LOAD_NAME                         0    (a)
+  1:5   -   1:9         2 LOAD_DOUBLE_O                     0    (12.0)
+  1:0   -   1:9         4 BINARY_OP                         1    (INPLACE_ADD)
+  1:0   -   1:1         6 STORE_NAME                        0    (a)
+  1:0   -   1:9         8 LOAD_NONE
+  1:0   -   1:9         9 RETURN_VALUE