[GR-34605] Fixes for Pillow

PullRequest: graalpython/2101

Author: msimacek

CHANGELOG.md

@@ -5,7 +5,7 @@ language runtime. The main focus is on user-observable behavior of the engine.
 
 ## Version 22.1.0
 * String conversion (`__str__`) now calls `toString` for Java objects and `toDisplayString` interop message for foreign objects.
-* Improved compatibility with PyPI package `lxml`
+* Improved compatibility with PyPI packages `lxml`, `pytz`, `Pillow`
 
 ## Version 22.0.0
 * Added support for `pyexpat` module.

graalpython/com.oracle.graal.python.cext/src/capi.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -42,6 +42,7 @@
 
 void *PY_TRUFFLE_CEXT;
 void *PY_BUILTIN;
+void *PY_SYS;
 void *Py_NoValue;
 
 void*(*pytruffle_decorate_function)(void *fun0, void* fun1);
@@ -72,6 +73,7 @@ __attribute__((constructor (__COUNTER__)))
 static void initialize_upcall_functions() {
     PY_TRUFFLE_CEXT = (void*)polyglot_eval("python", "import python_cext\npython_cext");
     PY_BUILTIN = (void*)polyglot_eval("python", "import builtins\nbuiltins");
+    PY_SYS = (void*)polyglot_eval("python", "import sys\nsys");
 
     pytruffle_decorate_function = ((void*(*)(void *fun0, void* fun1))polyglot_get_member(PY_TRUFFLE_CEXT, polyglot_from_string("PyTruffle_Decorate_Function", SRC_CS)));
 
@@ -322,13 +324,18 @@ static void initialize_bufferprocs() {
     polyglot_invoke(PY_TRUFFLE_CEXT, "PyTruffle_SetBufferProcs", native_to_java((PyObject*)&PyMemoryView_Type), (getbufferproc)memoryview_getbuffer, (releasebufferproc)memoryview_releasebuffer);
 }
 
+static void initialize_filesystemencoding() {
+    Py_FileSystemDefaultEncoding = (const char *)to_sulong(polyglot_invoke(PY_SYS, "getfilesystemencoding"));
+}
+
 __attribute__((constructor (20000)))
 static void initialize_capi() {
     // initialize global variables like '_Py_NoneStruct', etc.
     initialize_globals();
     initialize_exceptions();
     initialize_hashes();
     initialize_bufferprocs();
+    initialize_filesystemencoding();
 }
 
 // Workaround: use 'uint64' to avoid conversion to an LLVM boxed primitive such

graalpython/com.oracle.graal.python.cext/src/longobject.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -164,11 +164,10 @@ PyObject * PyLong_FromUnsignedLongLong(unsigned long long n) {
     return _jls_PyLong_FromLongLong(n, 0);
 }
 
-typedef PyObject* (*fromVoidPtr_fun_t)(void*);
-UPCALL_TYPED_ID(PyLong_FromVoidPtr, fromVoidPtr_fun_t);
+typedef PyObject* (*fromVoidPtr_fun_t)(void*, int32_t);
 PyObject * PyLong_FromVoidPtr(void *p) {
 	// directly do the upcall to avoid a cast to primitive and reference counting
-    return _jls_PyLong_FromVoidPtr(p);
+    return ((fromVoidPtr_fun_t)_jls_PyLong_FromLongLong)(p, 0);
 }
 
 UPCALL_ID(PyLong_AsVoidPtr);

graalpython/com.oracle.graal.python.cext/src/preconfig.c

@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "capi.h"
+#include "fileobject.h"
+
+const char *Py_FileSystemDefaultEncoding = NULL;

graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/parser/BasicTests.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -705,6 +705,16 @@ public void try11() throws Exception {
                                         "        print(err)");
     }
 
+    @Test
+    public void tryPrint() throws Exception {
+        checkScopeAndTree(
+                        "try:\n" +
+                                        "  1\n" +
+                                        "except:\n" +
+                                        "  pass",
+                        PythonParser.ParserMode.Statement);
+    }
+
     @Test
     public void tuple01() throws Exception {
         checkTreeResult("(1, 2, 3)");
@@ -924,6 +934,14 @@ public void with06() throws Exception {
                                         "      pass");
     }
 
+    @Test
+    public void withPrint() throws Exception {
+        checkScopeAndTree(
+                        "with A():\n" +
+                                        "  1",
+                        PythonParser.ParserMode.Statement);
+    }
+
     @Test
     public void spaceEnd() throws Exception {
         checkTreeResult("x=5 ");

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_method.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -43,6 +43,15 @@
 __dir__ = __file__.rpartition("/")[0]
 
 
+def assert_raises(err, fn, *args, **kwargs):
+    raised = False
+    try:
+        fn(*args, **kwargs)
+    except err:
+        raised = True
+    assert raised
+
+
 def _reference_classmethod(args):
     if isinstance(args[0], type(list.append)):
         return classmethod(args[0])()
@@ -103,6 +112,16 @@ def test_methods(self):
         assert obj.meth_static_varargs(1, 2, 3) == (None, (1, 2, 3))
         assert obj.meth_static_varargs_keywords(1, 2, 3, a=1, b=2) == (None, (1, 2, 3), {'a': 1, 'b': 2})
 
+        assert_raises(TypeError, obj.meth_noargs, 1)
+        assert_raises(TypeError, obj.meth_o)
+        assert_raises(TypeError, obj.meth_o, 1, 2)
+        assert_raises(TypeError, obj.meth_class_noargs, 1)
+        assert_raises(TypeError, obj.meth_class_o)
+        assert_raises(TypeError, obj.meth_class_o, 1, 2)
+        assert_raises(TypeError, obj.meth_static_noargs, 1)
+        assert_raises(TypeError, obj.meth_static_o)
+        assert_raises(TypeError, obj.meth_static_o, 1, 2)
+
 
 class TestPyMethod(CPyExtTestCase):
     def compile_module(self, name):

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_mmap.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -85,3 +85,30 @@ def compile_module(self, name):
         callfunction="get_mmap_buf",
         cmpfunc=unhandled_error_compare
     )
+
+    # Exercises conversion to native and copying from the actual mmap pointer
+    test_buffer_memcpy = CPyExtFunction(
+        lambda args: b"hello, world",
+        lambda: (
+            (create_and_map_file(),),
+        ),
+        code="""
+        static PyObject* get_mmap_buf(PyObject* mmapObj) {
+            Py_buffer buf;
+            Py_ssize_t len, i;
+            char* data = NULL;
+            if (PyObject_GetBuffer(mmapObj, &buf, PyBUF_SIMPLE)) {
+                return NULL;
+            }
+            len = buf.len;
+            data = (char*) malloc(sizeof(char)*len);
+            memcpy(data, buf.buf, len);
+            return PyBytes_FromStringAndSize(data, len);
+        }
+        """,
+        resultspec="O",
+        argspec='O',
+        arguments=["PyObject* mmapObj"],
+        callfunction="get_mmap_buf",
+        cmpfunc=unhandled_error_compare
+    )

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_modsupport.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -61,15 +61,25 @@ def _reference_typecheck(args, expected_type):
 
 
 def _reference_parse_O(args):
-    assert isinstance(args[0], tuple)
-    assert isinstance(args[1], dict)
-    if args[0]:
+    if not isinstance(args[0], tuple) or not isinstance(args[1], dict):
+        raise SystemError
+    if len(args[0]) == 1:
         return args[0][0]
     elif "arg0" in args[1]:
         return args[1]["arg0"]
     raise TypeError
 
 
+def _reference_parse_tuple(args):
+    try:
+        t = args[0][0]
+        if len(t) != 2:
+            raise TypeError
+        return t[0], t[1]
+    except Exception:
+        raise TypeError
+
+
 class Indexable:
     def __int__(self):
         return 456
@@ -78,6 +88,26 @@ def __index__(self):
         return 123
 
 
+class MySeq:
+    def __len__(self):
+        return 2
+
+    def __getitem__(self, item):
+        if item == 0:
+            return 'x'
+        elif item == 1:
+            return 'y'
+        else:
+            raise IndexError
+
+
+class BadSeq:
+    def __len__(self):
+        return 2
+
+    def __getitem__(self, item):
+        raise IndexError
+
 class TestModsupport(CPyExtTestCase):
     def compile_module(self, name):
         type(self).mro()[1].__dict__["test_%s" % name].create_module(name)
@@ -130,6 +160,9 @@ def compile_module(self, name):
             (tuple(), {"arg0": 'helloworld'}),
             (tuple(), dict()),
             (tuple(), {"arg1": 'helloworld'}),
+            (1, dict()),
+            (tuple(), 1),
+            (("a", "excess"), dict()),
         ),
         code='''
         static PyObject* wrap_PyArg_ParseTupleAndKeywords(PyObject* argTuple, PyObject* kwargs) {
@@ -149,6 +182,34 @@ def compile_module(self, name):
         cmpfunc=unhandled_error_compare
     )
 
+    test_parseargs_tuple = CPyExtFunction(
+        _reference_parse_tuple,
+        lambda: (
+            ((("a", "b"),),),
+            ((["a", "b"],),),
+            ((MySeq(),),),
+            ((["a"],),),
+            ((["a", "b", "c"],),),
+            ((1,),),
+            ((BadSeq(),),),
+        ),
+        code='''
+        static PyObject* wrap_PyArg_ParseTuple(PyObject* argTuple) {
+            PyObject* a = NULL;
+            PyObject* b = NULL;
+            if (PyArg_ParseTuple(argTuple, "(OO)", &a, &b) == 0) {
+                return NULL;
+            }
+            return Py_BuildValue("(OO)", a, b);
+        }
+        ''',
+        resultspec="O",
+        argspec="O",
+        arguments=["PyObject* argTuple"],
+        callfunction="wrap_PyArg_ParseTuple",
+        cmpfunc=unhandled_error_compare
+    )
+
     test_parseargs_O_conv = CPyExtFunction(
         lambda args: True if args[0][0] else False,
         lambda: (

graalpython/com.oracle.graal.python.test/testData/goldenFiles/BasicTests/tryPrint.scope

@@ -0,0 +1,5 @@
+Scope: []
+    Kind: Module
+    FrameDescriptor: Empty
+    CellVars: Empty
+    FreeVars: Empty

graalpython/com.oracle.graal.python.test/testData/goldenFiles/BasicTests/tryPrint.tast

@@ -0,0 +1,27 @@
+ModuleRootNode Name: <module> SourceSection: [0,23]`try:↵  1↵except:↵  p...`
+  Signature: varArgs=False, varKeywordArgs=False, noArguments=True, positionalOnly=True, requiresKeywordArgs=False
+  FreeVars: None
+  NeedsCellFrame: False
+  FrameDescriptor:  Empty
+  Documentation: None
+  InnerRootNode SourceSection: [0,23]`try:↵  1↵except:↵  p...`
+    ExpressionWithSideEffect SourceSection: [0,23]`try:↵  1↵except:↵  p...`
+      Expression:
+        EmptyNode SourceSection: None
+      SideEffect:
+        TryFinallyNode SourceSection: [0,23]`try:↵  1↵except:↵  p...`
+          TryExceptNode SourceSection: None
+            ExpressionStatementNode SourceSection: None
+              PrintExpressionNode SourceSection: None
+                GetAttributeNode SourceSection: None
+                  GetFixedAttributeNodeGen SourceSection: None
+                    Key: displayhook
+                    LookupAndCallNonReversibleBinaryNodeGen SourceSection: None
+                      Op: __getattribute__
+                CallNodeGen SourceSection: None
+                IntegerLiteralNode SourceSection: [7,8]`1`
+                  Value: 1
+            ExceptNode SourceSection: None
+              ExpressionStatementNode SourceSection: [19,23]`pass`
+                EmptyNode SourceSection: [19,23]`pass`
+            BlockNode SourceSection: None