From 3369142a7d5f2fd294a14fe45d7bc8baba2d00ef Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Sat, 19 Sep 2026 19:47:05 +0100 Subject: [PATCH 1/3] Manage Python thread states per Julia task --- AGENTS.md | 5 +- Project.toml | 2 +- docs/src/juliacall.md | 23 +-- docs/src/pythoncall-reference.md | 11 +- docs/src/pythoncall.md | 35 ++--- pysrc/juliacall/juliapkg-dev.json | 2 +- pysrc/juliacall/juliapkg.json | 2 +- src/API/exports.jl | 2 + src/API/macros.jl | 2 + src/C/context.jl | 9 +- src/C/pointers.jl | 9 ++ src/Core/Core.jl | 2 + src/Core/Py.jl | 26 ++- src/Core/builtins.jl | 78 +++++---- src/Core/err.jl | 40 ++--- src/GC/GC.jl | 21 ++- src/GIL/GIL.jl | 47 +++--- src/JlWrap/any.jl | 13 +- src/JlWrap/base.jl | 8 +- src/PyMacro/PyMacro.jl | 18 ++- src/PythonCall.jl | 5 + src/Region/Region.jl | 253 ++++++++++++++++++++++++++++++ test/GC.jl | 4 + test/Region.jl | 42 +++++ 24 files changed, 493 insertions(+), 166 deletions(-) create mode 100644 src/Region/Region.jl create mode 100644 test/Region.jl diff --git a/AGENTS.md b/AGENTS.md index 354305bd..236a5279 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,7 +5,10 @@ - **Python tests**: - Copy `pysrc/juliacall/juliapkg-dev.json` to `pysrc/juliacall/juliapkg.json` before running (do **not** commit this copy). - Execute with `uv run pytest -s --nbval ./pytest` (add `--cov=pysrc` when coverage is needed). - - Sometimes `juliapkg` requires Julia 1.10–1.11; `juliaup` already provides 1.11.7 in this environment. +- Sometimes `juliapkg` requires Julia 1.10–1.11; `juliaup` already provides 1.11.7 in this environment. +- With Python 3.14, juliapkg's OpenSSL compatibility currently constrains Julia to 1.11 or + older. Packages requiring Julia 1.12+ therefore cannot run the Python suite under that + Python; use an older Python whose OpenSSL constraint permits Julia 1.12+. The majority of tests live in the Julia package; Python tests cover functionality that cannot be exercised from Julia (e.g., JuliaCall-specific behavior). Run both suites—typically Julia first—in whichever order makes sense. diff --git a/Project.toml b/Project.toml index 9e6d15fe..970b2162 100644 --- a/Project.toml +++ b/Project.toml @@ -34,7 +34,7 @@ PyCall = "1" Serialization = "1" Tables = "1" UnsafePointers = "1" -julia = "1.10" +julia = "1.12" [extras] CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" diff --git a/docs/src/juliacall.md b/docs/src/juliacall.md index b6c073b0..8deb7f07 100644 --- a/docs/src/juliacall.md +++ b/docs/src/juliacall.md @@ -157,13 +157,11 @@ caveats. Most importantly, you can only call Python code while Python's [Global Interpreter Lock (GIL)](https://docs.python.org/3/glossary.html#term-global-interpreter-lock) -is locked by the current thread. You can use JuliaCall from any Python thread, and the GIL -will be locked whenever any JuliaCall function is used. However, to leverage the benefits -of multi-threading, you can unlock the GIL while executing any Julia code that does not -interact with Python. +JuliaCall borrows the Python thread state which entered Julia and automatically detaches it +while arbitrary Julia code runs. Nested Python interaction from that Julia code temporarily +reattaches the same state, and the borrowed state is restored before returning to Python. -The simplest way to do this is using the `_jl_call_nogil` method on Julia functions to -call the function with the GIL unlocked. +The historical `_jl_call_nogil` spelling remains available as a compatibility alias: ```python from concurrent.futures import ThreadPoolExecutor, wait @@ -173,16 +171,9 @@ fs = [pool.submit(jl.Libc.systemsleep._jl_call_nogil, 5) for _ in range(4)] wait(fs) ``` -In the above example, we call `Libc.systemsleep(5)` on four threads. Because we -called it with `_jl_call_nogil`, the GIL was unlocked, allowing the threads to run in -parallel, taking about 5 seconds in total. - -If we did not use `_jl_call_nogil` (i.e. if we did `pool.submit(jl.Libc.systemsleep, 5)`) -then the above code will take 20 seconds because the sleeps run one after another. - -It is very important that any function called with `_jl_call_nogil` does not interact -with Python at all unless it re-locks the GIL first, such as by using -[PythonCall.GIL.@lock](@ref). +Ordinary calls provide the same automatic resource management, so +`pool.submit(jl.Libc.systemsleep, 5)` is preferred. PythonCall operations nested inside +Julia callbacks are safe without explicit region or lock calls. You can also use [multi-threading from Julia](@ref jl-multi-threading). diff --git a/docs/src/pythoncall-reference.md b/docs/src/pythoncall-reference.md index 6c13d3fb..9b6400f5 100644 --- a/docs/src/pythoncall-reference.md +++ b/docs/src/pythoncall-reference.md @@ -222,8 +222,15 @@ Py(x::MyType) = x.py ## Multi-threading -These functions are not exported. They support multi-threading of Python and/or Julia. -See also [`juliacall.AnyValue._jl_call_nogil`](@ref julia-wrappers). +PythonCall manages Python thread state automatically. These exported macros are optional +performance and concurrency hints; users normally do not need them for correctness. + +```@docs +@pyregion +@pyregionbreak +``` + +The older `PythonCall.GIL` names remain as compatibility aliases. ```@docs PythonCall.GIL.lock diff --git a/docs/src/pythoncall.md b/docs/src/pythoncall.md index 9bdc42a2..3659329f 100644 --- a/docs/src/pythoncall.md +++ b/docs/src/pythoncall.md @@ -471,22 +471,15 @@ See [Installing Python packages](@ref python-deps). Multi-threading support is experimental and can change without notice. -From v0.9.22, PythonCall supports multi-threading in Julia and/or Python, with some -caveats. - -Most importantly, you can only call Python code while Python's -[Global Interpreter Lock (GIL)](https://docs.python.org/3/glossary.html#term-global-interpreter-lock) -is locked by the current thread. Ordinarily, the GIL is locked by the main thread in Julia, -so if you want to run Python code on any other thread, you must unlock the GIL from the -main thread and then re-lock it while running any Python code on other threads. - -This is made possible by the macros [`PythonCall.GIL.@unlock`](@ref) and -[`PythonCall.GIL.@lock`](@ref) or the functions [`PythonCall.GIL.unlock`](@ref) and -[`PythonCall.GIL.lock`](@ref) with this pattern: +PythonCall APIs automatically establish the Python thread state they need, so ordinary +operations can be called from any Julia task or thread without explicit locking. The +optional [`@pyregion`](@ref) macro amortizes those transitions across straight-line, +Python-heavy work. Use [`@pyregionbreak`](@ref) around Julia-heavy code which deliberately +yields, waits, or blocks cooperatively: ```julia -PythonCall.GIL.@unlock Threads.@threads for i in 1:4 - PythonCall.GIL.@lock pyimport("time").sleep(5) +Threads.@threads for i in 1:4 + @pyregion pyimport("time").sleep(5) end ``` @@ -494,9 +487,10 @@ In the above example, we call `time.sleep(5)` four times in parallel. If Julia w started with at least four threads (`julia -t4`) then the above code will take about 5 seconds. -Both `@unlock` and `@lock` are important. If the GIL were not unlocked, then a deadlock -would occur when attempting to lock the already-locked GIL from the threads. If the GIL -were not re-locked, then Python would crash when interacting with it. +Both region macros nest arbitrarily, and neither is required for correctness. A nested +PythonCall operation inside `@pyregionbreak` temporarily re-enters Python automatically. +On a GIL-enabled Python, attaching a state can block that Julia worker while CPython +arbitrates access; free-threaded Python uses the same state-management machinery. With multiple Julia threads you need exactly one interactive thread, see the [FAQ](@ref faq-multi-threading). @@ -504,9 +498,8 @@ You can also use [multi-threading from Python](@ref py-multi-threading). ### Caveat: Garbage collection -If Julia's GC collects any Python objects from a thread where the GIL is not currently -locked, then those Python objects will not immediately be deleted. Instead they will be -queued to be deleted in a later GC pass. +If Julia's GC collects Python objects while no Python thread state is already attached, +those objects are queued rather than making a finalizer block while attaching a state. If you find you have many Python objects not being deleted, you can call -[`PythonCall.GC.gc()`](@ref) or `GC.gc()` while the GIL is locked to clear the queue. +[`PythonCall.GC.gc()`](@ref) or `GC.gc()` to clear the queue. diff --git a/pysrc/juliacall/juliapkg-dev.json b/pysrc/juliacall/juliapkg-dev.json index 3e6910a8..3dd2bbd1 100644 --- a/pysrc/juliacall/juliapkg-dev.json +++ b/pysrc/juliacall/juliapkg-dev.json @@ -1,5 +1,5 @@ { - "julia": "^1.10.3", + "julia": "^1.12", "packages": { "PythonCall": { "uuid": "6099a3de-0909-46bc-b1f4-468b9a2dfc0d", diff --git a/pysrc/juliacall/juliapkg.json b/pysrc/juliacall/juliapkg.json index f45ce3b1..37f68861 100644 --- a/pysrc/juliacall/juliapkg.json +++ b/pysrc/juliacall/juliapkg.json @@ -1,5 +1,5 @@ { - "julia": "^1.10.3", + "julia": "^1.12", "packages": { "PythonCall": { "uuid": "6099a3de-0909-46bc-b1f4-468b9a2dfc0d", diff --git a/src/API/exports.jl b/src/API/exports.jl index cdc5ac83..d2a048fc 100644 --- a/src/API/exports.jl +++ b/src/API/exports.jl @@ -3,6 +3,8 @@ export @py export @pyconst export @pyeval export @pyexec +export @pyregion +export @pyregionbreak export ispy export Py export pyabs diff --git a/src/API/macros.jl b/src/API/macros.jl index c74d191c..741ae191 100644 --- a/src/API/macros.jl +++ b/src/API/macros.jl @@ -2,6 +2,8 @@ macro pyconst end macro pyeval end macro pyexec end +macro pyregion end +macro pyregionbreak end # Convert macro pyconvert end diff --git a/src/C/context.jl b/src/C/context.jl index 6ccdab89..220bf88d 100644 --- a/src/C/context.jl +++ b/src/C/context.jl @@ -21,6 +21,11 @@ A handle to a loaded instance of libpython, its interpreter, function pointers, end const CTX = Context() +const FINALIZE_HOOK = Ref{Function}(() -> begin + if Py_FinalizeEx() == -1 + @warn "Py_FinalizeEx() error" + end +end) function _atpyexit() if CTX.is_initialized && !CTX.is_preinitialized @@ -282,9 +287,7 @@ function init_context() Py_InitializeEx(0) atexit() do CTX.is_initialized = false - if Py_FinalizeEx() == -1 - @warn "Py_FinalizeEx() error" - end + FINALIZE_HOOK[]() end end CTX.is_initialized = true diff --git a/src/C/pointers.jl b/src/C/pointers.jl index 9644329f..6108fa3b 100644 --- a/src/C/pointers.jl +++ b/src/C/pointers.jl @@ -19,6 +19,8 @@ const CAPI_FUNC_SIGS = Dict{Symbol,Pair{Tuple,Type}}( # GIL & THREADS :PyEval_SaveThread => () => Ptr{Cvoid}, :PyEval_RestoreThread => (Ptr{Cvoid},) => Cvoid, + :PyThreadState_New => (Ptr{Cvoid},) => Ptr{Cvoid}, + :PyThreadState_GetInterpreter => (Ptr{Cvoid},) => Ptr{Cvoid}, :PyGILState_Ensure => () => PyGILState_STATE, :PyGILState_Release => (PyGILState_STATE,) => Cvoid, :PyGILState_GetThisThreadState => () => Ptr{Cvoid}, @@ -278,6 +280,7 @@ const CAPI_OBJECTS = Set([ $([:($name::PyPtr = C_NULL) for name in CAPI_EXCEPTIONS]...) $([:($name::PyPtr = C_NULL) for name in CAPI_OBJECTS]...) PyOS_InputHookPtr::Ptr{Ptr{Cvoid}} = C_NULL + PyThreadState_GetUnchecked::Ptr{Cvoid} = C_NULL end const POINTERS = CAPIPointers() @@ -295,8 +298,14 @@ const POINTERS = CAPIPointers() ) $([:(p.$name = dlsym(lib, $(QuoteNode(name)))) for name in CAPI_OBJECTS]...) p.PyOS_InputHookPtr = dlsym(CTX.lib_ptr, :PyOS_InputHook) + p.PyThreadState_GetUnchecked = let q = dlsym_e(lib, :PyThreadState_GetUnchecked) + q == C_NULL ? dlsym(lib, :_PyThreadState_UncheckedGet) : q + end end +PyThreadState_GetUnchecked() = + ccall(POINTERS.PyThreadState_GetUnchecked, Ptr{Cvoid}, ()) + for (name, (argtypes, rettype)) in CAPI_FUNC_SIGS args = [Symbol("x", i) for (i, _) in enumerate(argtypes)] @eval $name($(args...)) = ccall(POINTERS.$name, $rettype, ($(argtypes...),), $(args...)) diff --git a/src/Core/Core.jl b/src/Core/Core.jl index 628643b6..ebb0becf 100644 --- a/src/Core/Core.jl +++ b/src/Core/Core.jl @@ -32,6 +32,8 @@ using Markdown: Markdown import ..PythonCall: @pyconst, + @pyregion, + @pyregionbreak, @pyeval, @pyexec, ispy, diff --git a/src/Core/Py.jl b/src/Core/Py.jl index 50ff0eb2..f86a1670 100644 --- a/src/Core/Py.jl +++ b/src/Core/Py.jl @@ -1,5 +1,5 @@ -incref(x::C.PyPtr) = (C.Py_IncRef(x); x) -decref(x::C.PyPtr) = (C.Py_DecRef(x); x) +incref(x::C.PyPtr) = @pyregion (C.Py_IncRef(x); x) +decref(x::C.PyPtr) = @pyregion (C.Py_DecRef(x); x) """ ispy(x) @@ -85,10 +85,12 @@ Use this to eagerly free a Python object, rather than waiting for Julia's GC to it at some indeterminate point in the future. """ function unsafe_pydel(x::Py) - ptr = getptr(x) - if ptr != C.PyNULL - C.Py_DecRef(ptr) - setptr!(x, C.PyNULL) + @pyregion begin + ptr = getptr(x) + if ptr != C.PyNULL + C.Py_DecRef(ptr) + setptr!(x, C.PyNULL) + end end return end @@ -99,12 +101,14 @@ macro autopy(args...) body = args[end] # ans = gensym("ans") esc(quote + @pyregion begin # $([:($t = $ispy($v) ? $v : $Py($v)) for (t, v) in zip(ts, vs)]...) # $ans = $body # $([:($ispy($v) || $unsafe_pydel($t)) for (t, v) in zip(ts, vs)]...) # $ans $([:($t = $Py($v)) for (t, v) in zip(ts, vs)]...) $body + end end) end @@ -291,15 +295,7 @@ function _propertynames(x::Py, private::Bool) return Symbol[Symbol(pystr_asstring(word)) for word in words] end -function Base.propertynames(x::Py, private::Bool = false) - if C.PyGILState_Check() == 1 - _propertynames(x, private) - else - C.on_main_thread() do - _propertynames(x, private) - end::Vector{Symbol} - end -end +Base.propertynames(x::Py, private::Bool = false) = @pyregion _propertynames(x, private) Base.Bool(x::Py) = pytruth(x) diff --git a/src/Core/builtins.jl b/src/Core/builtins.jl index 1bb634cb..13faca50 100644 --- a/src/Core/builtins.jl +++ b/src/Core/builtins.jl @@ -515,7 +515,7 @@ Returns the next item from the iterator `x`. If there are no more items, returns given, else raises `StopIteration`. """ function pynext(x) - ptr = errcheck_ambig(C.PyIter_Next(x)) + ptr = @pyregion errcheck_ambig(C.PyIter_Next(x)) if ptr == C.PyNULL errset(pybuiltins.StopIteration) pythrow() @@ -525,7 +525,7 @@ function pynext(x) end function pynext(x, d) - ptr = errcheck_ambig(C.PyIter_Next(x)) + ptr = @pyregion errcheck_ambig(C.PyIter_Next(x)) ptr == C.PyNULL ? d : pynew(ptr) end @@ -534,7 +534,8 @@ end Return the next item in the iterator `x`. When there are no more items, return NULL. """ -unsafe_pynext(x::Py) = Base.GC.@preserve x pynew(errcheck_ambig(C.PyIter_Next(x))) +unsafe_pynext(x::Py) = + @pyregion Base.GC.@preserve x pynew(errcheck_ambig(C.PyIter_Next(x))) ### None @@ -567,7 +568,8 @@ end ### str -pystr_fromUTF8(x::Ptr, n::Integer) = pynew(errcheck(C.PyUnicode_DecodeUTF8(x, n, C_NULL))) +pystr_fromUTF8(x::Ptr, n::Integer) = + @pyregion pynew(errcheck(C.PyUnicode_DecodeUTF8(x, n, C_NULL))) pystr_fromUTF8(x) = pystr_fromUTF8(pointer(x), sizeof(x)) """ @@ -583,23 +585,26 @@ pystr(x::AbstractString) = pystr(convert(String, x)::String) pystr(x::AbstractChar) = pystr(convert(Char, x)::Char) pystr(::Type{String}, x) = (s = pystr(x); ans = pystr_asstring(s); unsafe_pydel(s); ans) -pystr_asUTF8bytes(x::Py) = pynew(errcheck(C.PyUnicode_AsUTF8String(x))) +pystr_asUTF8bytes(x::Py) = @pyregion pynew(errcheck(C.PyUnicode_AsUTF8String(x))) pystr_asUTF8vector(x::Py) = (b = pystr_asUTF8bytes(x); ans = pybytes_asvector(b); unsafe_pydel(b); ans) pystr_asstring(x::Py) = (b = pystr_asUTF8bytes(x); ans = pybytes_asUTF8string(b); unsafe_pydel(b); ans) function pystr_intern!(x::Py) - ptr = Ref(getptr(x)) - C.PyUnicode_InternInPlace(ptr) - setptr!(x, ptr[]) + @pyregion begin + ptr = Ref(getptr(x)) + C.PyUnicode_InternInPlace(ptr) + setptr!(x, ptr[]) + end end pyisstr(x) = pytypecheckfast(x, C.Py_TPFLAGS_UNICODE_SUBCLASS) ### bytes -pybytes_fromdata(x::Ptr, n::Integer) = pynew(errcheck(C.PyBytes_FromStringAndSize(x, n))) +pybytes_fromdata(x::Ptr, n::Integer) = + @pyregion pynew(errcheck(C.PyBytes_FromStringAndSize(x, n))) pybytes_fromdata(x) = pybytes_fromdata(pointer(x), sizeof(x)) """ @@ -619,10 +624,12 @@ pybytes(::Type{T}, x) where {Base.CodeUnits{UInt8,String} <: T <: Base.CodeUnits pyisbytes(x) = pytypecheckfast(x, C.Py_TPFLAGS_BYTES_SUBCLASS) function pybytes_asdata(x::Py) - ptr = Ref(Ptr{Cchar}(0)) - len = Ref(C.Py_ssize_t(0)) - errcheck(C.PyBytes_AsStringAndSize(x, ptr, len)) - ptr[], len[] + @pyregion begin + ptr = Ref(Ptr{Cchar}(0)) + len = Ref(C.Py_ssize_t(0)) + errcheck(C.PyBytes_AsStringAndSize(x, ptr, len)) + ptr[], len[] + end end function pybytes_asvector(x::Py) @@ -648,19 +655,23 @@ pyint_fallback(x::Integer) = pyint_fallback(BigInt(x)) Convert `x` to a Python `int`. """ function pyint(x::Integer = 0) - y = mod(x, Clonglong) - if x == y - pynew(errcheck(C.PyLong_FromLongLong(y))) - else - pyint_fallback(x) + @pyregion begin + y = mod(x, Clonglong) + if x == y + pynew(errcheck(C.PyLong_FromLongLong(y))) + else + pyint_fallback(x) + end end end function pyint(x::Unsigned) - y = mod(x, Culonglong) - if x == y - pynew(errcheck(C.PyLong_FromUnsignedLongLong(y))) - else - pyint_fallback(x) + @pyregion begin + y = mod(x, Culonglong) + if x == y + pynew(errcheck(C.PyLong_FromUnsignedLongLong(y))) + else + pyint_fallback(x) + end end end pyint(x) = @autopy x pynew(errcheck(C.PyNumber_Long(x_))) @@ -674,7 +685,7 @@ pyisint(x) = pytypecheckfast(x, C.Py_TPFLAGS_LONG_SUBCLASS) Convert `x` to a Python `float`. """ -pyfloat(x::Real = 0.0) = pynew(errcheck(C.PyFloat_FromDouble(x))) +pyfloat(x::Real = 0.0) = @pyregion pynew(errcheck(C.PyFloat_FromDouble(x))) pyfloat(x) = @autopy x pynew(errcheck(C.PyNumber_Float(x_))) pyisfloat(x) = pytypecheck(x, pybuiltins.float) @@ -689,7 +700,8 @@ pyfloat_asdouble(x) = errcheck_ambig(@autopy x C.PyFloat_AsDouble(x_)) Convert `x` to a Python `complex`, or create one from given real and imaginary parts. """ -pycomplex(x::Real = 0.0, y::Real = 0.0) = pynew(errcheck(C.PyComplex_FromDoubles(x, y))) +pycomplex(x::Real = 0.0, y::Real = 0.0) = + @pyregion pynew(errcheck(C.PyComplex_FromDoubles(x, y))) pycomplex(x::Complex) = pycomplex(real(x), imag(x)) pycomplex(x) = pybuiltins.complex(x) pycomplex(x, y) = pybuiltins.complex(x, y) @@ -819,15 +831,15 @@ pyisrange(x) = pytypecheck(x, pybuiltins.range) ### tuple -pynulltuple(len) = pynew(errcheck(C.PyTuple_New(len))) +pynulltuple(len) = @pyregion pynew(errcheck(C.PyTuple_New(len))) function pytuple_setitem(xs::Py, i, x) - errcheck(C.PyTuple_SetItem(xs, i, incref(Py(x)))) + @pyregion errcheck(C.PyTuple_SetItem(xs, i, incref(Py(x)))) return xs end function pytuple_getitem(xs::Py, i) - Base.GC.@preserve xs pynew(incref(errcheck(C.PyTuple_GetItem(xs, i)))) + @pyregion Base.GC.@preserve xs pynew(incref(errcheck(C.PyTuple_GetItem(xs, i)))) end function pytuple_fromiter(xs) @@ -874,10 +886,10 @@ pyistuple(x) = pytypecheckfast(x, C.Py_TPFLAGS_TUPLE_SUBCLASS) ### list -pynulllist(len) = pynew(errcheck(C.PyList_New(len))) +pynulllist(len) = @pyregion pynew(errcheck(C.PyList_New(len))) function pylist_setitem(xs::Py, i, x) - errcheck(C.PyList_SetItem(xs, i, incref(Py(x)))) + @pyregion errcheck(C.PyList_SetItem(xs, i, incref(Py(x)))) return xs end @@ -972,7 +984,7 @@ Convert `x` to a Python `set`. If `x` is a Python object, this is equivalent to `set(x)` in Python. Otherwise `x` must be iterable. """ -pyset() = pynew(errcheck(C.PySet_New(C.PyNULL))) +pyset() = @pyregion pynew(errcheck(C.PySet_New(C.PyNULL))) pyset(x) = ispy(x) ? pybuiltins.set(x) : pyset_fromiter(x) """ @@ -983,7 +995,7 @@ Convert `x` to a Python `frozenset`. If `x` is a Python object, this is equivalent to `frozenset(x)` in Python. Otherwise `x` must be iterable. """ -pyfrozenset() = pynew(errcheck(C.PyFrozenSet_New(C.PyNULL))) +pyfrozenset() = @pyregion pynew(errcheck(C.PyFrozenSet_New(C.PyNULL))) pyfrozenset(x) = ispy(x) ? pybuiltins.frozenset(x) : pyfrozenset_fromiter(x) ### dict @@ -1017,7 +1029,7 @@ If `x` is a Python object, this is equivalent to `dict(x)` in Python. Otherwise `x` must iterate over key-value pairs. """ pydict(; kwargs...) = - isempty(kwargs) ? pynew(errcheck(C.PyDict_New())) : pystrdict_fromiter(kwargs) + isempty(kwargs) ? (@pyregion pynew(errcheck(C.PyDict_New()))) : pystrdict_fromiter(kwargs) pydict(x) = ispy(x) ? pybuiltins.dict(x) : pydict_fromiter(x) pydict(x::NamedTuple) = pydict(; x...) pydict(pair::Pair, pairs::Pair...) = pydict((pair, pairs...)) diff --git a/src/Core/err.jl b/src/Core/err.jl index 4ee00c74..8e49ab44 100644 --- a/src/Core/err.jl +++ b/src/Core/err.jl @@ -3,7 +3,7 @@ errval(::T) where {T<:Number} = zero(T) - one(T) iserrval(val) = val == errval(val) -iserrset() = C.PyErr_Occurred() != C.PyNULL +iserrset() = @pyregion C.PyErr_Occurred() != C.PyNULL iserrset(val) = val == errval(val) errcheck() = iserrset() ? pythrow() : nothing @@ -13,31 +13,35 @@ iserrset_ambig(val) = iserrset(val) && iserrset() errcheck_ambig(val) = iserrset_ambig(val) ? pythrow() : val -errclear() = C.PyErr_Clear() +errclear() = @pyregion C.PyErr_Clear() errmatches(t) = (@autopy t C.PyErr_ExceptionMatches(t_)) == 1 function errget() - t = Ref(C.PyNULL) - v = Ref(C.PyNULL) - b = Ref(C.PyNULL) - C.PyErr_Fetch(t, v, b) - (pynew(t[]), pynew(v[]), pynew(b[])) + @pyregion begin + t = Ref(C.PyNULL) + v = Ref(C.PyNULL) + b = Ref(C.PyNULL) + C.PyErr_Fetch(t, v, b) + (pynew(t[]), pynew(v[]), pynew(b[])) + end end -errset(t::Py) = C.PyErr_SetNone(t) -errset(t::Py, v::Py) = C.PyErr_SetObject(t, v) -errset(t::Py, v::String) = C.PyErr_SetString(t, v) +errset(t::Py) = @pyregion C.PyErr_SetNone(t) +errset(t::Py, v::Py) = @pyregion C.PyErr_SetObject(t, v) +errset(t::Py, v::String) = @pyregion C.PyErr_SetString(t, v) function errnormalize!(t::Py, v::Py, b::Py) - tref = Ref(getptr(t)) - vref = Ref(getptr(v)) - bref = Ref(getptr(b)) - C.PyErr_NormalizeException(tref, vref, bref) - setptr!(t, tref[]) - setptr!(v, vref[]) - setptr!(b, bref[]) - (t, v, b) + @pyregion begin + tref = Ref(getptr(t)) + vref = Ref(getptr(v)) + bref = Ref(getptr(b)) + C.PyErr_NormalizeException(tref, vref, bref) + setptr!(t, tref[]) + setptr!(v, vref[]) + setptr!(b, bref[]) + (t, v, b) + end end function PyException(v::Py = pybuiltins.None) diff --git a/src/GC/GC.jl b/src/GC/GC.jl index 67c43565..5655bf6a 100644 --- a/src/GC/GC.jl +++ b/src/GC/GC.jl @@ -8,6 +8,8 @@ See [`gc`](@ref). module GC using ..C: C +using ..Region +import ..PythonCall: @pyregion if Base.VERSION ≥ v"1.11" eval( @@ -64,15 +66,12 @@ end Free any Python objects waiting to be freed. -These are objects that were finalized from a thread that was not holding the Python -GIL at the time. - -Like most PythonCall functions, this must only be called from the main thread (i.e. the -thread currently holding the Python GIL.) +These are objects finalized while no Python thread state was attached. This explicit +operation safely establishes the state it needs and can be called from any Julia thread. """ function gc() if C.CTX.is_initialized - unsafe_free_queue() + @pyregion unsafe_free_queue() end nothing end @@ -94,8 +93,8 @@ function enqueue(ptr::C.PyPtr) # If C.CTX.is_initialized is false then the Python interpreter hasn't started yet # or has been finalized; either way attempting to free will cause an error. if ptr != C.PyNULL && C.CTX.is_initialized - if C.PyGILState_Check() == 1 - # If the current thread holds the GIL, then we can immediately free. + if Region.has_tstate() + # An attached state lets us immediately free without blocking a finalizer. C.Py_DecRef(ptr) # We may as well also free any other enqueued objects. if !isempty(QUEUE.items) @@ -103,7 +102,7 @@ function enqueue(ptr::C.PyPtr) end else # Otherwise we push the pointer onto the queue to be freed later, either: - # (a) If a future Python object is finalized on the thread holding the GIL + # (a) If a future Python object is finalized with a state already attached # in the branch above. # (b) If the GCHook() object below is finalized in an ordinary GC. # (c) If the user calls PythonCall.GC.gc(). @@ -115,7 +114,7 @@ end function enqueue_all(ptrs) if any(!=(C.PyNULL), ptrs) && C.CTX.is_initialized - if C.PyGILState_Check() == 1 + if Region.has_tstate() for ptr in ptrs if ptr != C.PyNULL C.Py_DecRef(ptr) @@ -150,7 +149,7 @@ end function _gchook_finalizer(x) if C.CTX.is_initialized finalizer(_gchook_finalizer, x) - if !isempty(QUEUE.items) && C.PyGILState_Check() == 1 + if !isempty(QUEUE.items) && Region.has_tstate() unsafe_free_queue() end end diff --git a/src/GIL/GIL.jl b/src/GIL/GIL.jl index f4b386ce..115e1135 100644 --- a/src/GIL/GIL.jl +++ b/src/GIL/GIL.jl @@ -1,7 +1,7 @@ """ module PythonCall.GIL -Handling the Python Global Interpreter Lock. +Compatibility API for Python interaction regions. See [`lock`](@ref), [`@lock`](@ref), [`unlock`](@ref) and [`@unlock`](@ref). @@ -11,7 +11,7 @@ See [`lock`](@ref), [`@lock`](@ref), [`unlock`](@ref) and [`@unlock`](@ref). """ module GIL -using ..C: C +using ..Region if Base.VERSION ≥ v"1.11" eval( @@ -29,11 +29,10 @@ end """ lock(f) -Lock the GIL, compute `f()`, unlock the GIL, then return the result of `f()`. +Compute `f()` in a Python-heavy region and return its result. -Use this to run Python code from threads that do not currently hold the GIL, such as new -threads. Since the main Julia thread holds the GIL by default, you will need to -[`unlock`](@ref) the GIL before using this function. +PythonCall APIs already establish the required Python thread state automatically. This +compatibility function can amortize transitions across several operations. See [`@lock`](@ref) for the macro form. @@ -42,22 +41,18 @@ See [`@lock`](@ref) for the macro form. This function is experimental. Its semantics may be changed without notice. """ function lock(f) - state = C.PyGILState_Ensure() + token = Region.enter_region() try f() finally - C.PyGILState_Release(state) + Region.exit_region(token) end end """ @lock expr -Lock the GIL, compute `expr`, unlock the GIL, then return the result of `expr`. - -Use this to run Python code from threads that do not currently hold the GIL, such as new -threads. Since the main Julia thread holds the GIL by default, you will need to -[`@unlock`](@ref) the GIL before using this function. +Compute `expr` in a Python-heavy region and return its result. The macro equivalent of [`lock`](@ref). @@ -67,11 +62,11 @@ The macro equivalent of [`lock`](@ref). """ macro lock(expr) quote - state = C.PyGILState_Ensure() + token = $Region.enter_region() try $(esc(expr)) finally - C.PyGILState_Release(state) + $Region.exit_region(token) end end end @@ -79,11 +74,8 @@ end """ unlock(f) -Unlock the GIL, compute `f()`, re-lock the GIL, then return the result of `f()`. - -Use this to run non-Python code with the GIL unlocked, so allowing another thread to run -Python code. That other thread can be a Julia thread, which must lock the GIL using -[`lock`](@ref). +Temporarily relinquish Python-related resources, compute `f()`, restore them, and return +the result. Prefer [`PythonCall.@pyregionbreak`](@ref) in new code. See [`@unlock`](@ref) for the macro form. @@ -92,22 +84,19 @@ See [`@unlock`](@ref) for the macro form. This function is experimental. Its semantics may be changed without notice. """ function unlock(f) - state = C.PyEval_SaveThread() + token = Region.enter_break() try f() finally - C.PyEval_RestoreThread(state) + Region.exit_break(token) end end """ @unlock expr -Unlock the GIL, compute `expr`, re-lock the GIL, then return the result of `expr`. - -Use this to run non-Python code with the GIL unlocked, so allowing another thread to run -Python code. That other thread can be a Julia thread, which must lock the GIL using -[`@lock`](@ref). +Temporarily relinquish Python-related resources, compute `expr`, restore them, and return +the result. Prefer [`PythonCall.@pyregionbreak`](@ref) in new code. The macro equivalent of [`unlock`](@ref). @@ -117,11 +106,11 @@ The macro equivalent of [`unlock`](@ref). """ macro unlock(expr) quote - state = C.PyEval_SaveThread() + token = $Region.enter_break() try $(esc(expr)) finally - C.PyEval_RestoreThread(state) + $Region.exit_break(token) end end end diff --git a/src/JlWrap/any.jl b/src/JlWrap/any.jl index a0df83e2..e58c2aa6 100644 --- a/src/JlWrap/any.jl +++ b/src/JlWrap/any.jl @@ -94,12 +94,12 @@ function pyjlany_call_nogil(self, args_::Py, kwargs_::Py) if pylen(kwargs_) > 0 args = pyconvert(Vector{Any}, args_) kwargs = pyconvert(Dict{Symbol,Any}, kwargs_) - ans = pyjl(GIL.@unlock self(args...; kwargs...)) + ans = pyjl(self(args...; kwargs...)) elseif pylen(args_) > 0 args = pyconvert(Vector{Any}, args_) - ans = pyjl(GIL.@unlock self(args...)) + ans = pyjl(self(args...)) else - ans = pyjl(GIL.@unlock self()) + ans = pyjl(self()) end unsafe_pydel(args_) unsafe_pydel(kwargs_) @@ -608,11 +608,8 @@ class Jl(JlBase2): def jl_callback(self, *args, **kwargs): return self._jl_callmethod($(pyjl_methodnum(pyjlany_callback)), args, kwargs) def jl_call_nogil(self, *args, **kwargs): - '''Call this with the given arguments but with the GIL disabled. - - WARNING: This function must not interact with Python at all without re-acquiring - the GIL. - ''' + '''Compatibility alias for calling this Julia object. Python resources are + relinquished automatically while Julia code runs.''' return self._jl_callmethod($(pyjl_methodnum(pyjlany_call_nogil)), args, kwargs) def _repr_mimebundle_(self, include=None, exclude=None): return self._jl_callmethod($(pyjl_methodnum(pyjlany_mimebundle)), include, exclude) diff --git a/src/JlWrap/base.jl b/src/JlWrap/base.jl index 2497f4b0..67723e59 100644 --- a/src/JlWrap/base.jl +++ b/src/JlWrap/base.jl @@ -39,25 +39,25 @@ function Cjl._pyjl_callmethod(f, self_::C.PyPtr, args_::C.PyPtr, nargs::C.Py_ssi try if nargs == 1 in_f = true - ans = f(self)::Py + ans = @pyregionbreak(f(self))::Py in_f = false elseif nargs == 2 arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) in_f = true - ans = f(self, arg1)::Py + ans = @pyregionbreak(f(self, arg1))::Py in_f = false elseif nargs == 3 arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) arg2 = pynew(incref(C.PyTuple_GetItem(args_, 2))) in_f = true - ans = f(self, arg1, arg2)::Py + ans = @pyregionbreak(f(self, arg1, arg2))::Py in_f = false elseif nargs == 4 arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) arg2 = pynew(incref(C.PyTuple_GetItem(args_, 2))) arg3 = pynew(incref(C.PyTuple_GetItem(args_, 3))) in_f = true - ans = f(self, arg1, arg2, arg3)::Py + ans = @pyregionbreak(f(self, arg1, arg2, arg3))::Py in_f = false else errset( diff --git a/src/PyMacro/PyMacro.jl b/src/PyMacro/PyMacro.jl index d8dc45ab..a185ea8e 100644 --- a/src/PyMacro/PyMacro.jl +++ b/src/PyMacro/PyMacro.jl @@ -560,7 +560,7 @@ function py_macro_lower(st, body, ans, ex; flavour = :expr) # @jl x elseif @capture(ex, @jl ax_) y = py_macro_lower_jl(st, ax) - py_macro_assign(body, ans, y) + py_macro_assign(body, ans, :(@pyregionbreak $y)) return false # @compile code mode=mode ... @@ -918,7 +918,21 @@ Py({'y': 2}) This macro is experimental. It may be modified or removed in a future release. """ macro py(ex) - esc(py_macro(ex, __module__, __source__)) + lowered = py_macro(ex, __module__, __source__) + assigned = Symbol[] + function find_assigned(x) + x isa Expr || return + if x.head === :(=) && x.args[1] isa Symbol + push!(assigned, x.args[1]) + end + foreach(find_assigned, x.args) + end + find_assigned(lowered) + locals = unique(assigned) + esc(quote + $(isempty(locals) ? nothing : Expr(:local, locals...)) + @pyregion $lowered + end) end end diff --git a/src/PythonCall.jl b/src/PythonCall.jl index c6b166bd..48da84ba 100644 --- a/src/PythonCall.jl +++ b/src/PythonCall.jl @@ -6,6 +6,7 @@ include("API/API.jl") include("Utils/Utils.jl") include("NumpyDates/NumpyDates.jl") include("C/C.jl") +include("Region/Region.jl") include("GIL/GIL.jl") include("GC/GC.jl") include("Core/Core.jl") @@ -15,6 +16,10 @@ include("Wrap/Wrap.jl") include("JlWrap/JlWrap.jl") include("Compat/Compat.jl") +function __init__() + Region.start_runtime() +end + # not API but used in tests for k in [ :pyjlanytype, diff --git a/src/Region/Region.jl b/src/Region/Region.jl new file mode 100644 index 00000000..72ff5871 --- /dev/null +++ b/src/Region/Region.jl @@ -0,0 +1,253 @@ +"""Internal task-safe management of CPython thread states.""" +module Region + +using ..C: C +import ..PythonCall: @pyregion, @pyregionbreak + +const PyThreadStatePtr = Ptr{Cvoid} + +mutable struct ThreadState + sem::Base.Semaphore + tstate::PyThreadStatePtr +end +ThreadState() = ThreadState(Base.Semaphore(1), C_NULL) + +mutable struct TaskState + tstate::PyThreadStatePtr + sem::Union{Nothing,Base.Semaphore} + attached::Bool + tid::Int + oldsticky::Bool +end +TaskState() = TaskState(C_NULL, nothing, false, 0, false) + +const THREAD_STATE = Base.OncePerThread(ThreadState) +const TASK_STATE = Base.OncePerTask(TaskState) +const INTERP = Ref{Ptr{Cvoid}}(C_NULL) + +current_tstate() = C.PyThreadState_GetUnchecked() +has_tstate() = C.CTX.is_initialized && current_tstate() != C_NULL + +function reset!(s::TaskState) + s.tstate = C_NULL + s.sem = nothing + s.attached = false + s.tid = 0 + return +end + +function start_session!(s::TaskState) + task = current_task() + s.oldsticky = task.sticky + task.sticky = true + s.tid = Threads.threadid() + ts = THREAD_STATE() + s.sem = ts.sem + return ts +end + +check_thread(s::TaskState) = (@assert current_task().sticky && Threads.threadid() == s.tid) + +function enter_region() + s = TASK_STATE() + if s.tstate != C_NULL + check_thread(s) + if s.attached + return :noop + end + Base.acquire(s.sem::Base.Semaphore) + try + C.PyEval_RestoreThread(s.tstate) + s.attached = true + catch + Base.release(s.sem::Base.Semaphore) + rethrow() + end + return :detach + end + + task = current_task() + ts = try + start_session!(s) + catch + task.sticky = s.oldsticky + reset!(s) + rethrow() + end + acquired = false + try + Base.acquire(ts.sem) + acquired = true + current = current_tstate() + if current != C_NULL + s.tstate = current + s.attached = true + return :root_borrowed + end + if ts.tstate == C_NULL + ts.tstate = C.PyThreadState_New(INTERP[]) + ts.tstate == C_NULL && error("PyThreadState_New failed") + end + s.tstate = ts.tstate + C.PyEval_RestoreThread(s.tstate) + s.attached = true + return :root_owned + catch + acquired && Base.release(ts.sem) + task.sticky = s.oldsticky + reset!(s) + rethrow() + end +end + +function exit_region(token) + token === :noop && return + s = TASK_STATE() + check_thread(s) + if token === :detach + saved = C.PyEval_SaveThread() + @assert saved == s.tstate + s.attached = false + Base.release(s.sem::Base.Semaphore) + elseif token === :root_owned + saved = C.PyEval_SaveThread() + @assert saved == s.tstate + sem, oldsticky = s.sem::Base.Semaphore, s.oldsticky + reset!(s) + Base.release(sem) + current_task().sticky = oldsticky + elseif token === :root_borrowed + @assert current_tstate() == s.tstate + sem, oldsticky = s.sem::Base.Semaphore, s.oldsticky + reset!(s) + Base.release(sem) + current_task().sticky = oldsticky + else + error("invalid Python region token") + end + return +end + +function enter_break() + s = TASK_STATE() + if s.tstate != C_NULL + check_thread(s) + !s.attached && return :noop + saved = C.PyEval_SaveThread() + @assert saved == s.tstate + s.attached = false + Base.release(s.sem::Base.Semaphore) + return :restore + end + current_tstate() == C_NULL && return :noop + + task = current_task() + ts = try + start_session!(s) + catch + task.sticky = s.oldsticky + reset!(s) + rethrow() + end + acquired = false + try + Base.acquire(ts.sem) + acquired = true + current = current_tstate() + if current == C_NULL + Base.release(ts.sem) + task.sticky = s.oldsticky + reset!(s) + return :noop + end + s.tstate = current + saved = C.PyEval_SaveThread() + @assert saved == current + s.attached = false + Base.release(ts.sem) + return :root_restore + catch + acquired && Base.release(ts.sem) + task.sticky = s.oldsticky + reset!(s) + rethrow() + end +end + +function exit_break(token) + token === :noop && return + s = TASK_STATE() + check_thread(s) + Base.acquire(s.sem::Base.Semaphore) + C.PyEval_RestoreThread(s.tstate) + s.attached = true + if token === :root_restore + sem, oldsticky = s.sem::Base.Semaphore, s.oldsticky + reset!(s) + Base.release(sem) + current_task().sticky = oldsticky + elseif token !== :restore + error("invalid Python region-break token") + end + return +end + +""" + @pyregion expr + +Mark `expr` as relatively straight-line, Python-heavy work. PythonCall APIs manage the +Python resources they need automatically; this optional region can amortize that work across +several operations. Regions nest freely. Put Julia code which deliberately yields, waits, or +blocks cooperatively in [`@pyregionbreak`](@ref). +""" +macro pyregion(ex) + quote + local token = $enter_region() + try + $(esc(ex)) + finally + $exit_region(token) + end + end +end + +""" + @pyregionbreak expr + +Mark `expr` as a section where Python interaction is absent or infrequent, allowing an +enclosing [`@pyregion`](@ref) to relinquish Python-related resources temporarily. Nested +PythonCall operations still work automatically, and both kinds of region nest freely. +""" +macro pyregionbreak(ex) + quote + local token = $enter_break() + try + $(esc(ex)) + finally + $exit_break(token) + end + end +end + +function __init__() + current = current_tstate() + current == C_NULL && error("Python initialization did not leave an attached thread state") + INTERP[] = C.PyThreadState_GetInterpreter(current) + INTERP[] == C_NULL && error("could not determine the Python interpreter") + return +end + +function start_runtime() + if !C.CTX.is_embedded && !C.CTX.is_preinitialized + C.PyEval_SaveThread() + C.FINALIZE_HOOK[] = function () + ts = THREAD_STATE() + ts.tstate == C_NULL && (ts.tstate = C.PyThreadState_New(INTERP[])) + C.PyEval_RestoreThread(ts.tstate) + C.Py_FinalizeEx() == -1 && @warn "Py_FinalizeEx() error" + end + end + return +end + +end diff --git a/test/GC.jl b/test/GC.jl index f180cfaf..f305c6f7 100644 --- a/test/GC.jl +++ b/test/GC.jl @@ -29,5 +29,9 @@ end VERSION >= v"1.10.0-" && @test !isempty(PythonCall.GC.QUEUE.items) GC.gc() + # A Julia finalizer must never attach a Python thread state merely to decref. + @test PythonCall.C.PyThreadState_GetUnchecked() == C_NULL + Threads.nthreads() > 1 && @test !isempty(PythonCall.GC.QUEUE.items) + PythonCall.GC.gc() @test isempty(PythonCall.GC.QUEUE.items) end diff --git a/test/Region.jl b/test/Region.jl new file mode 100644 index 00000000..b000ad52 --- /dev/null +++ b/test/Region.jl @@ -0,0 +1,42 @@ +using TestItemRunner + +@testitem "Python regions and task-safe thread states" setup = [Setup] begin + using Base.Threads + + @test PythonCall.C.PyThreadState_GetUnchecked() == C_NULL + @test @pyregion pyconvert(Int, pyint(12)) == 12 + @test PythonCall.C.PyThreadState_GetUnchecked() == C_NULL + + @test @pyregion begin + @pyregion pyconvert(Int, pyint(1)) == 1 + @pyregionbreak begin + yield() + @pyregion pyconvert(Int, pyint(2)) == 2 + @pyregionbreak yield() + end + pyconvert(Int, pyint(3)) == 3 + end + + @test_throws ErrorException @pyregion @pyregionbreak error("region exception") + @test PythonCall.C.PyThreadState_GetUnchecked() == C_NULL + @test_throws PyException @pyregion pybuiltins.int("not an integer") + @test pyconvert(Int, pyint(5)) == 5 + + results = fetch.([@spawn begin + total = 0 + for i in 1:100 + total += pyconvert(Int, pyint(i)) + @pyregionbreak yield() + end + total + end for _ in 1:max(8, 2nthreads())]) + @test all(==(5050), results) + + f = pyfunc() do + yield() + pyconvert(Int, pybuiltins.sum([1, 2, 3])) + end + @test pyconvert(Int, f()) == 6 + + @test pyconvert(Int, @py 1 + @jl(pyconvert(Int, pyint(2)))) == 3 +end From d35fa84d4e01767b2f981898348775133fec9abe Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Sat, 19 Sep 2026 22:08:30 +0100 Subject: [PATCH 2/3] Restore Julia 1.10 thread state compatibility --- AGENTS.md | 3 --- Project.toml | 2 +- pysrc/juliacall/juliapkg-dev.json | 2 +- pysrc/juliacall/juliapkg.json | 2 +- pytest/test_all.py | 2 ++ src/Region/Region.jl | 5 +++-- src/Utils/Utils.jl | 36 +++++++++++++++++++++++++++++++ test/Utils.jl | 23 ++++++++++++++++++++ 8 files changed, 67 insertions(+), 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 236a5279..ce85f833 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,9 +6,6 @@ - Copy `pysrc/juliacall/juliapkg-dev.json` to `pysrc/juliacall/juliapkg.json` before running (do **not** commit this copy). - Execute with `uv run pytest -s --nbval ./pytest` (add `--cov=pysrc` when coverage is needed). - Sometimes `juliapkg` requires Julia 1.10–1.11; `juliaup` already provides 1.11.7 in this environment. -- With Python 3.14, juliapkg's OpenSSL compatibility currently constrains Julia to 1.11 or - older. Packages requiring Julia 1.12+ therefore cannot run the Python suite under that - Python; use an older Python whose OpenSSL constraint permits Julia 1.12+. The majority of tests live in the Julia package; Python tests cover functionality that cannot be exercised from Julia (e.g., JuliaCall-specific behavior). Run both suites—typically Julia first—in whichever order makes sense. diff --git a/Project.toml b/Project.toml index 970b2162..9e6d15fe 100644 --- a/Project.toml +++ b/Project.toml @@ -34,7 +34,7 @@ PyCall = "1" Serialization = "1" Tables = "1" UnsafePointers = "1" -julia = "1.12" +julia = "1.10" [extras] CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" diff --git a/pysrc/juliacall/juliapkg-dev.json b/pysrc/juliacall/juliapkg-dev.json index 3dd2bbd1..3e6910a8 100644 --- a/pysrc/juliacall/juliapkg-dev.json +++ b/pysrc/juliacall/juliapkg-dev.json @@ -1,5 +1,5 @@ { - "julia": "^1.12", + "julia": "^1.10.3", "packages": { "PythonCall": { "uuid": "6099a3de-0909-46bc-b1f4-468b9a2dfc0d", diff --git a/pysrc/juliacall/juliapkg.json b/pysrc/juliacall/juliapkg.json index 37f68861..f45ce3b1 100644 --- a/pysrc/juliacall/juliapkg.json +++ b/pysrc/juliacall/juliapkg.json @@ -1,5 +1,5 @@ { - "julia": "^1.12", + "julia": "^1.10.3", "packages": { "PythonCall": { "uuid": "6099a3de-0909-46bc-b1f4-468b9a2dfc0d", diff --git a/pytest/test_all.py b/pytest/test_all.py index 7ed9fbb5..0267f7df 100644 --- a/pytest/test_all.py +++ b/pytest/test_all.py @@ -105,6 +105,8 @@ def test_julia_gc(): end end GC.gc() + @test !isempty(PythonCall.GC.QUEUE.items) + PythonCall.GC.gc() @test isempty(PythonCall.GC.QUEUE.items) """ ) diff --git a/src/Region/Region.jl b/src/Region/Region.jl index 72ff5871..d69e1fab 100644 --- a/src/Region/Region.jl +++ b/src/Region/Region.jl @@ -2,6 +2,7 @@ module Region using ..C: C +using ..Utils import ..PythonCall: @pyregion, @pyregionbreak const PyThreadStatePtr = Ptr{Cvoid} @@ -21,8 +22,8 @@ mutable struct TaskState end TaskState() = TaskState(C_NULL, nothing, false, 0, false) -const THREAD_STATE = Base.OncePerThread(ThreadState) -const TASK_STATE = Base.OncePerTask(TaskState) +const THREAD_STATE = Utils.OncePerThread{ThreadState}(ThreadState) +const TASK_STATE = Utils.OncePerTask{TaskState}(TaskState) const INTERP = Ref{Ptr{Cvoid}}(C_NULL) current_tstate() = C.PyThreadState_GetUnchecked() diff --git a/src/Utils/Utils.jl b/src/Utils/Utils.jl index ea10a115..be5d2a5a 100644 --- a/src/Utils/Utils.jl +++ b/src/Utils/Utils.jl @@ -2,6 +2,42 @@ module Utils using Preferences: @load_preference +# Minimal package-local equivalents of Base.OncePerThread and Base.OncePerTask. The Base +# implementations are only available on newer Julia releases, while PythonCall supports +# Julia 1.10. These intentionally provide only the callable interface needed internally. +mutable struct OncePerThread{T,F} + initializer::F + values::Dict{Int,T} + lock::ReentrantLock +end + +OncePerThread{T}(initializer::F) where {T,F} = + OncePerThread{T,F}(initializer, Dict{Int,T}(), ReentrantLock()) + +function (once::OncePerThread{T})() where {T} + tid = Threads.threadid() + lock(once.lock) + try + return get!(once.values, tid) do + once.initializer()::T + end + finally + unlock(once.lock) + end +end + +mutable struct OncePerTask{T,F} + initializer::F +end + +OncePerTask{T}(initializer::F) where {T,F} = OncePerTask{T,F}(initializer) + +function (once::OncePerTask{T})() where {T} + get!(task_local_storage(), once) do + once.initializer()::T + end::T +end + function getpref(::Type{T}, prefname, envname, default = nothing) where {T} ans = @load_preference(prefname, nothing) ans === nothing || return checkpref(T, ans)::T diff --git a/test/Utils.jl b/test/Utils.jl index 33c82bd9..3b49a28d 100644 --- a/test/Utils.jl +++ b/test/Utils.jl @@ -22,3 +22,26 @@ end @test s[1:2] == "ab" @test s[1:2:end] == "aaaab" end + +@testitem "OncePerThread and OncePerTask" begin + thread_count = Threads.Atomic{Int}(0) + per_thread = PythonCall.Utils.OncePerThread{Int}() do + Threads.atomic_add!(thread_count, 1) + Threads.threadid() + end + @test per_thread() == Threads.threadid() + @test per_thread() == Threads.threadid() + @test thread_count[] == 1 + + task_count = Threads.Atomic{Int}(0) + per_task = PythonCall.Utils.OncePerTask{UInt}() do + Threads.atomic_add!(task_count, 1) + objectid(current_task()) + end + @test per_task() == per_task() + other_per_task = PythonCall.Utils.OncePerTask{UInt}(() -> typemax(UInt)) + @test other_per_task() == typemax(UInt) + other = fetch(Threads.@spawn (per_task(), per_task())) + @test other[1] == other[2] + @test task_count[] == 2 +end From 8f82c46fb6672dae947da1aa2aec85b8f94f63d1 Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Mon, 21 Sep 2026 09:18:38 +0100 Subject: [PATCH 3/3] Keep Python error handling within regions --- src/Convert/pyconvert.jl | 49 ++++--- src/Core/builtins.jl | 274 +++++++++++++++++++++------------------ src/Core/err.jl | 60 +++++---- src/JlWrap/any.jl | 10 +- src/JlWrap/array.jl | 40 +++--- src/JlWrap/base.jl | 106 +++++++-------- src/JlWrap/io.jl | 10 +- src/JlWrap/set.jl | 6 +- src/JlWrap/vector.jl | 12 +- 9 files changed, 299 insertions(+), 268 deletions(-) diff --git a/src/Convert/pyconvert.jl b/src/Convert/pyconvert.jl index 10435f11..ab69aad9 100644 --- a/src/Convert/pyconvert.jl +++ b/src/Convert/pyconvert.jl @@ -369,13 +369,15 @@ On failure, evaluates to `onfail`, which defaults to `return pyconvert_unconvert """ macro pyconvert(T, x, onfail = :(return $pyconvert_unconverted())) quote - T = $(esc(T)) - x = $(esc(x)) - ans = pytryconvert(T, x) - if pyconvert_isunconverted(ans) - $(esc(onfail)) - else - pyconvert_result(T, ans) + @pyregion begin + T = $(esc(T)) + x = $(esc(x)) + ans = pytryconvert(T, x) + if pyconvert_isunconverted(ans) + $(esc(onfail)) + else + pyconvert_result(T, ans) + end end end end @@ -387,10 +389,19 @@ Convert the Python object `x` to a `T`. If `d` is specified, it is returned on failure instead of throwing an error. """ -pyconvert(::Type{T}, x) where {T} = @autopy x @pyconvert T x_ error( - "cannot convert this Python '$(pytype(x_).__name__)' to a Julia '$T'", -) -pyconvert(::Type{T}, x, d) where {T} = @autopy x @pyconvert T x_ d +function pyconvert(::Type{T}, x) where {T} + @pyregion begin + @autopy x @pyconvert T x_ error( + "cannot convert this Python '$(pytype(x_).__name__)' to a Julia '$T'", + ) + end +end + +function pyconvert(::Type{T}, x, d) where {T} + @pyregion begin + @autopy x @pyconvert T x_ d + end +end """ pyconvertarg(T, x, name) @@ -399,12 +410,16 @@ Convert the Python object `x` to a `T`. On failure, throws a Python `TypeError` saying that the argument `name` could not be converted. """ -pyconvertarg(::Type{T}, x, name) where {T} = @autopy x @pyconvert T x_ begin - errset( - pybuiltins.TypeError, - "Cannot convert argument '$name' to a Julia '$T', got a '$(pytype(x_).__name__)'", - ) - pythrow() +function pyconvertarg(::Type{T}, x, name) where {T} + @pyregion begin + @autopy x @pyconvert T x_ begin + errset( + pybuiltins.TypeError, + "Cannot convert argument '$name' to a Julia '$T', got a '$(pytype(x_).__name__)'", + ) + pythrow() + end + end end function init_pyconvert() diff --git a/src/Core/builtins.jl b/src/Core/builtins.jl index 13faca50..29a07e43 100644 --- a/src/Core/builtins.jl +++ b/src/Core/builtins.jl @@ -5,7 +5,7 @@ True if `x` and `y` are the same Python object. Equivalent to `x is y` in Python. """ -pyis(x, y) = @autopy x y getptr(x_) == getptr(y_) +pyis(x, y) = @pyregion @autopy x y getptr(x_) == getptr(y_) pyisnot(x, y) = !pyis(x, y) @@ -14,7 +14,7 @@ pyisnot(x, y) = !pyis(x, y) Equivalent to `repr(x)` in Python. """ -pyrepr(x) = pynew(errcheck(@autopy x C.PyObject_Repr(x_))) +pyrepr(x) = @pyregion pynew(errcheck(@autopy x C.PyObject_Repr(x_))) pyrepr(::Type{String}, x) = (s = pyrepr(x); ans = pystr_asstring(s); unsafe_pydel(s); ans) """ @@ -22,7 +22,7 @@ pyrepr(::Type{String}, x) = (s = pyrepr(x); ans = pystr_asstring(s); unsafe_pyde Equivalent to `ascii(x)` in Python. """ -pyascii(x) = pynew(errcheck(@autopy x C.PyObject_ASCII(x_))) +pyascii(x) = @pyregion pynew(errcheck(@autopy x C.PyObject_ASCII(x_))) pyascii(::Type{String}, x) = (s = pyascii(x); ans = pystr_asstring(s); unsafe_pydel(s); ans) """ @@ -33,17 +33,19 @@ Equivalent to `hasattr(x, k)` in Python. Tests if `getattr(x, k)` raises an `AttributeError`. """ function pyhasattr(x, k) - ptr = @autopy x k C.PyObject_GetAttr(x_, k_) - if iserrset(ptr) - if errmatches(pybuiltins.AttributeError) - errclear() - return false + @pyregion begin + ptr = @autopy x k C.PyObject_GetAttr(x_, k_) + if iserrset(ptr) + if errmatches(pybuiltins.AttributeError) + errclear() + return false + else + pythrow() + end else - pythrow() + decref(ptr) + return true end - else - decref(ptr) - return true end end # pyhasattr(x, k) = errcheck(@autopy x k C.PyObject_HasAttr(x_, k_)) == 1 @@ -55,18 +57,20 @@ Equivalent to `getattr(x, k)` or `x.k` in Python. If `d` is specified, it is returned if the attribute does not exist. """ -pygetattr(x, k) = pynew(errcheck(@autopy x k C.PyObject_GetAttr(x_, k_))) +pygetattr(x, k) = @pyregion pynew(errcheck(@autopy x k C.PyObject_GetAttr(x_, k_))) function pygetattr(x, k, d) - ptr = @autopy x k C.PyObject_GetAttr(x_, k_) - if iserrset(ptr) - if errmatches(pybuiltins.AttributeError) - errclear() - return d + @pyregion begin + ptr = @autopy x k C.PyObject_GetAttr(x_, k_) + if iserrset(ptr) + if errmatches(pybuiltins.AttributeError) + errclear() + return d + else + pythrow() + end else - pythrow() + return pynew(ptr) end - else - return pynew(ptr) end end @@ -75,56 +79,56 @@ end Equivalent to `setattr(x, k, v)` or `x.k = v` in Python. """ -pysetattr(x, k, v) = (errcheck(@autopy x k v C.PyObject_SetAttr(x_, k_, v_)); nothing) +pysetattr(x, k, v) = @pyregion (errcheck(@autopy x k v C.PyObject_SetAttr(x_, k_, v_)); nothing) """ pydelattr(x, k) Equivalent to `delattr(x, k)` or `del x.k` in Python. """ -pydelattr(x, k) = (errcheck(@autopy x k C.PyObject_SetAttr(x_, k_, C.PyNULL)); nothing) +pydelattr(x, k) = @pyregion (errcheck(@autopy x k C.PyObject_SetAttr(x_, k_, C.PyNULL)); nothing) """ pyissubclass(s, t) Test if `s` is a subclass of `t`. Equivalent to `issubclass(s, t)` in Python. """ -pyissubclass(s, t) = errcheck(@autopy s t C.PyObject_IsSubclass(s_, t_)) == 1 +pyissubclass(s, t) = @pyregion errcheck(@autopy s t C.PyObject_IsSubclass(s_, t_)) == 1 """ pyisinstance(x, t) Test if `x` is of type `t`. Equivalent to `isinstance(x, t)` in Python. """ -pyisinstance(x, t) = errcheck(@autopy x t C.PyObject_IsInstance(x_, t_)) == 1 +pyisinstance(x, t) = @pyregion errcheck(@autopy x t C.PyObject_IsInstance(x_, t_)) == 1 """ pyhash(x) Equivalent to `hash(x)` in Python, converted to an `Integer`. """ -pyhash(x) = errcheck(@autopy x C.PyObject_Hash(x_)) +pyhash(x) = @pyregion errcheck(@autopy x C.PyObject_Hash(x_)) """ pytruth(x) The truthyness of `x`. Equivalent to `bool(x)` in Python, converted to a `Bool`. """ -pytruth(x) = errcheck(@autopy x C.PyObject_IsTrue(x_)) == 1 +pytruth(x) = @pyregion errcheck(@autopy x C.PyObject_IsTrue(x_)) == 1 """ pynot(x) The falsyness of `x`. Equivalent to `not x` in Python, converted to a `Bool`. """ -pynot(x) = errcheck(@autopy x C.PyObject_Not(x_)) == 1 +pynot(x) = @pyregion errcheck(@autopy x C.PyObject_Not(x_)) == 1 """ pylen(x) The length of `x`. Equivalent to `len(x)` in Python, converted to an `Integer`. """ -pylen(x) = errcheck(@autopy x C.PyObject_Length(x_)) +pylen(x) = @pyregion errcheck(@autopy x C.PyObject_Length(x_)) """ pyhasitem(x, k) @@ -132,17 +136,19 @@ pylen(x) = errcheck(@autopy x C.PyObject_Length(x_)) Test if `pygetitem(x, k)` raises a `KeyError` or `AttributeError`. """ function pyhasitem(x, k) - ptr = @autopy x k C.PyObject_GetItem(x_, k_) - if iserrset(ptr) - if errmatches(pybuiltins.KeyError) || errmatches(pybuiltins.IndexError) - errclear() - return false + @pyregion begin + ptr = @autopy x k C.PyObject_GetItem(x_, k_) + if iserrset(ptr) + if errmatches(pybuiltins.KeyError) || errmatches(pybuiltins.IndexError) + errclear() + return false + else + pythrow() + end else - pythrow() + decref(ptr) + return true end - else - decref(ptr) - return true end end @@ -154,18 +160,20 @@ Equivalent `x[k]` in Python. If `d` is specified, it is returned if the item does not exist (i.e. if `x[k]` raises a `KeyError` or `IndexError`). """ -pygetitem(x, k) = pynew(errcheck(@autopy x k C.PyObject_GetItem(x_, k_))) +pygetitem(x, k) = @pyregion pynew(errcheck(@autopy x k C.PyObject_GetItem(x_, k_))) function pygetitem(x, k, d) - ptr = @autopy x k C.PyObject_GetItem(x_, k_) - if iserrset(ptr) - if errmatches(pybuiltins.KeyError) || errmatches(pybuiltins.IndexError) - errclear() - return d + @pyregion begin + ptr = @autopy x k C.PyObject_GetItem(x_, k_) + if iserrset(ptr) + if errmatches(pybuiltins.KeyError) || errmatches(pybuiltins.IndexError) + errclear() + return d + else + pythrow() + end else - pythrow() + return pynew(ptr) end - else - return pynew(ptr) end end @@ -174,26 +182,26 @@ end Equivalent to `setitem(x, k, v)` or `x[k] = v` in Python. """ -pysetitem(x, k, v) = (errcheck(@autopy x k v C.PyObject_SetItem(x_, k_, v_)); nothing) +pysetitem(x, k, v) = @pyregion (errcheck(@autopy x k v C.PyObject_SetItem(x_, k_, v_)); nothing) """ pydelitem(x, k) Equivalent to `delitem(x, k)` or `del x[k]` in Python. """ -pydelitem(x, k) = (errcheck(@autopy x k C.PyObject_DelItem(x_, k_)); nothing) +pydelitem(x, k) = @pyregion (errcheck(@autopy x k C.PyObject_DelItem(x_, k_)); nothing) """ pydir(x) Equivalent to `dir(x)` in Python. """ -pydir(x) = pynew(errcheck(@autopy x C.PyObject_Dir(x_))) +pydir(x) = @pyregion pynew(errcheck(@autopy x C.PyObject_Dir(x_))) -pycallargs(f) = pynew(errcheck(@autopy f C.PyObject_CallObject(f_, C.PyNULL))) -pycallargs(f, args) = pynew(errcheck(@autopy f args C.PyObject_CallObject(f_, args_))) +pycallargs(f) = @pyregion pynew(errcheck(@autopy f C.PyObject_CallObject(f_, C.PyNULL))) +pycallargs(f, args) = @pyregion pynew(errcheck(@autopy f args C.PyObject_CallObject(f_, args_))) pycallargs(f, args, kwargs) = - pynew(errcheck(@autopy f args kwargs C.PyObject_Call(f_, args_, kwargs_))) + @pyregion pynew(errcheck(@autopy f args kwargs C.PyObject_Call(f_, args_, kwargs_))) """ pycall(f, args...; kwargs...) @@ -223,7 +231,7 @@ pycall(f, args...; kwargs...) = Equivalent to `x == y` in Python. The second form converts to `Bool`. """ -pyeq(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_EQ))) +pyeq(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_EQ))) """ pyne(x, y) @@ -231,7 +239,7 @@ pyeq(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_EQ)) Equivalent to `x != y` in Python. The second form converts to `Bool`. """ -pyne(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_NE))) +pyne(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_NE))) """ pyle(x, y) @@ -239,7 +247,7 @@ pyne(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_NE)) Equivalent to `x <= y` in Python. The second form converts to `Bool`. """ -pyle(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LE))) +pyle(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LE))) """ pylt(x, y) @@ -247,7 +255,7 @@ pyle(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LE)) Equivalent to `x < y` in Python. The second form converts to `Bool`. """ -pylt(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LT))) +pylt(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LT))) """ pyge(x, y) @@ -255,7 +263,7 @@ pylt(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LT)) Equivalent to `x >= y` in Python. The second form converts to `Bool`. """ -pyge(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_GE))) +pyge(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_GE))) """ pygt(x, y) @@ -263,26 +271,26 @@ pyge(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_GE)) Equivalent to `x > y` in Python. The second form converts to `Bool`. """ -pygt(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_GT))) +pygt(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_GT))) pyeq(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_EQ)) == 1 + @pyregion errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_EQ)) == 1 pyne(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_NE)) == 1 + @pyregion errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_NE)) == 1 pyle(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_LE)) == 1 + @pyregion errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_LE)) == 1 pylt(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_LT)) == 1 + @pyregion errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_LT)) == 1 pyge(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_GE)) == 1 + @pyregion errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_GE)) == 1 pygt(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_GT)) == 1 + @pyregion errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_GT)) == 1 """ pycontains(x, v) Equivalent to `v in x` in Python. """ -pycontains(x, v) = errcheck(@autopy x v C.PySequence_Contains(x_, v_)) == 1 +pycontains(x, v) = @pyregion errcheck(@autopy x v C.PySequence_Contains(x_, v_)) == 1 """ pyin(v, x) @@ -301,31 +309,31 @@ pynotin(v, x) = !pyin(v, x) Equivalent to `-x` in Python. """ -pyneg(x) = pynew(errcheck(@autopy x C.PyNumber_Negative(x_))) +pyneg(x) = @pyregion pynew(errcheck(@autopy x C.PyNumber_Negative(x_))) """ pypos(x) Equivalent to `+x` in Python. """ -pypos(x) = pynew(errcheck(@autopy x C.PyNumber_Positive(x_))) +pypos(x) = @pyregion pynew(errcheck(@autopy x C.PyNumber_Positive(x_))) """ pyabs(x) Equivalent to `abs(x)` in Python. """ -pyabs(x) = pynew(errcheck(@autopy x C.PyNumber_Absolute(x_))) +pyabs(x) = @pyregion pynew(errcheck(@autopy x C.PyNumber_Absolute(x_))) """ pyinv(x) Equivalent to `~x` in Python. """ -pyinv(x) = pynew(errcheck(@autopy x C.PyNumber_Invert(x_))) +pyinv(x) = @pyregion pynew(errcheck(@autopy x C.PyNumber_Invert(x_))) """ pyindex(x) Convert `x` losslessly to an `int`. """ -pyindex(x) = pynew(errcheck(@autopy x C.PyNumber_Index(x_))) +pyindex(x) = @pyregion pynew(errcheck(@autopy x C.PyNumber_Index(x_))) # binary """ @@ -333,79 +341,79 @@ pyindex(x) = pynew(errcheck(@autopy x C.PyNumber_Index(x_))) Equivalent to `x + y` in Python. """ -pyadd(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Add(x_, y_))) +pyadd(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Add(x_, y_))) """ pysub(x, y) Equivalent to `x - y` in Python. """ -pysub(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Subtract(x_, y_))) +pysub(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Subtract(x_, y_))) """ pymul(x, y) Equivalent to `x * y` in Python. """ -pymul(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Multiply(x_, y_))) +pymul(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Multiply(x_, y_))) """ pymatmul(x, y) Equivalent to `x @ y` in Python. """ -pymatmul(x, y) = pynew(errcheck(@autopy x y C.PyNumber_MatrixMultiply(x_, y_))) +pymatmul(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_MatrixMultiply(x_, y_))) """ pyfloordiv(x, y) Equivalent to `x // y` in Python. """ -pyfloordiv(x, y) = pynew(errcheck(@autopy x y C.PyNumber_FloorDivide(x_, y_))) +pyfloordiv(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_FloorDivide(x_, y_))) """ pytruediv(x, y) Equivalent to `x / y` in Python. """ -pytruediv(x, y) = pynew(errcheck(@autopy x y C.PyNumber_TrueDivide(x_, y_))) +pytruediv(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_TrueDivide(x_, y_))) """ pymod(x, y) Equivalent to `x % y` in Python. """ -pymod(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Remainder(x_, y_))) +pymod(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Remainder(x_, y_))) """ pydivmod(x, y) Equivalent to `divmod(x, y)` in Python. """ -pydivmod(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Divmod(x_, y_))) +pydivmod(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Divmod(x_, y_))) """ pylshift(x, y) Equivalent to `x << y` in Python. """ -pylshift(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Lshift(x_, y_))) +pylshift(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Lshift(x_, y_))) """ pyrshift(x, y) Equivalent to `x >> y` in Python. """ -pyrshift(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Rshift(x_, y_))) +pyrshift(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Rshift(x_, y_))) """ pyand(x, y) Equivalent to `x & y` in Python. """ -pyand(x, y) = pynew(errcheck(@autopy x y C.PyNumber_And(x_, y_))) +pyand(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_And(x_, y_))) """ pyxor(x, y) Equivalent to `x ^ y` in Python. """ -pyxor(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Xor(x_, y_))) +pyxor(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Xor(x_, y_))) """ pyor(x, y) Equivalent to `x | y` in Python. """ -pyor(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Or(x_, y_))) +pyor(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Or(x_, y_))) # binary in-place """ @@ -413,73 +421,73 @@ pyor(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Or(x_, y_))) In-place add. `x = pyiadd(x, y)` is equivalent to `x += y` in Python. """ -pyiadd(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceAdd(x_, y_))) +pyiadd(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceAdd(x_, y_))) """ pyisub(x, y) In-place subtract. `x = pyisub(x, y)` is equivalent to `x -= y` in Python. """ -pyisub(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceSubtract(x_, y_))) +pyisub(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceSubtract(x_, y_))) """ pyimul(x, y) In-place multiply. `x = pyimul(x, y)` is equivalent to `x *= y` in Python. """ -pyimul(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceMultiply(x_, y_))) +pyimul(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceMultiply(x_, y_))) """ pyimatmul(x, y) In-place matrix multiply. `x = pyimatmul(x, y)` is equivalent to `x @= y` in Python. """ -pyimatmul(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceMatrixMultiply(x_, y_))) +pyimatmul(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceMatrixMultiply(x_, y_))) """ pyifloordiv(x, y) In-place floor divide. `x = pyifloordiv(x, y)` is equivalent to `x //= y` in Python. """ -pyifloordiv(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceFloorDivide(x_, y_))) +pyifloordiv(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceFloorDivide(x_, y_))) """ pyitruediv(x, y) In-place true division. `x = pyitruediv(x, y)` is equivalent to `x /= y` in Python. """ -pyitruediv(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceTrueDivide(x_, y_))) +pyitruediv(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceTrueDivide(x_, y_))) """ pyimod(x, y) In-place subtraction. `x = pyimod(x, y)` is equivalent to `x %= y` in Python. """ -pyimod(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceRemainder(x_, y_))) +pyimod(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceRemainder(x_, y_))) """ pyilshift(x, y) In-place left shift. `x = pyilshift(x, y)` is equivalent to `x <<= y` in Python. """ -pyilshift(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceLshift(x_, y_))) +pyilshift(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceLshift(x_, y_))) """ pyirshift(x, y) In-place right shift. `x = pyirshift(x, y)` is equivalent to `x >>= y` in Python. """ -pyirshift(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceRshift(x_, y_))) +pyirshift(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceRshift(x_, y_))) """ pyiand(x, y) In-place and. `x = pyiand(x, y)` is equivalent to `x &= y` in Python. """ -pyiand(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceAnd(x_, y_))) +pyiand(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceAnd(x_, y_))) """ pyixor(x, y) In-place xor. `x = pyixor(x, y)` is equivalent to `x ^= y` in Python. """ -pyixor(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceXor(x_, y_))) +pyixor(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceXor(x_, y_))) """ pyior(x, y) In-place or. `x = pyior(x, y)` is equivalent to `x |= y` in Python. """ -pyior(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceOr(x_, y_))) +pyior(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceOr(x_, y_))) # power """ @@ -488,14 +496,14 @@ pyior(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceOr(x_, y_))) Equivalent to `x ** y` or `pow(x, y, z)` in Python. """ pypow(x, y, z = pybuiltins.None) = - pynew(errcheck(@autopy x y z C.PyNumber_Power(x_, y_, z_))) + @pyregion pynew(errcheck(@autopy x y z C.PyNumber_Power(x_, y_, z_))) """ pyipow(x, y, z=None) In-place power. `x = pyipow(x, y)` is equivalent to `x **= y` in Python. """ pyipow(x, y, z = pybuiltins.None) = - pynew(errcheck(@autopy x y z C.PyNumber_InPlacePower(x_, y_, z_))) + @pyregion pynew(errcheck(@autopy x y z C.PyNumber_InPlacePower(x_, y_, z_))) ### iter @@ -504,7 +512,7 @@ pyipow(x, y, z = pybuiltins.None) = Equivalent to `iter(x)` in Python. """ -pyiter(x) = pynew(errcheck(@autopy x C.PyObject_GetIter(x_))) +pyiter(x) = @pyregion pynew(errcheck(@autopy x C.PyObject_GetIter(x_))) """ pynext(x, [d]) @@ -515,18 +523,22 @@ Returns the next item from the iterator `x`. If there are no more items, returns given, else raises `StopIteration`. """ function pynext(x) - ptr = @pyregion errcheck_ambig(C.PyIter_Next(x)) - if ptr == C.PyNULL - errset(pybuiltins.StopIteration) - pythrow() - else - pynew(ptr) + @pyregion begin + ptr = errcheck_ambig(C.PyIter_Next(x)) + if ptr == C.PyNULL + errset(pybuiltins.StopIteration) + pythrow() + else + pynew(ptr) + end end end function pynext(x, d) - ptr = @pyregion errcheck_ambig(C.PyIter_Next(x)) - ptr == C.PyNULL ? d : pynew(ptr) + @pyregion begin + ptr = errcheck_ambig(C.PyIter_Next(x)) + ptr == C.PyNULL ? d : pynew(ptr) + end end """ @@ -577,7 +589,7 @@ pystr_fromUTF8(x) = pystr_fromUTF8(pointer(x), sizeof(x)) Convert `x` to a Python `str`. """ -pystr(x) = pynew(errcheck(@autopy x C.PyObject_Str(x_))) +pystr(x) = @pyregion pynew(errcheck(@autopy x C.PyObject_Str(x_))) pystr(x::String) = pystr_fromUTF8(x) pystr(x::SubString{String}) = pystr_fromUTF8(x) pystr(x::Char) = pystr(string(x)) @@ -612,7 +624,7 @@ pybytes_fromdata(x) = pybytes_fromdata(pointer(x), sizeof(x)) Convert `x` to a Python `bytes`. """ -pybytes(x) = pynew(errcheck(@autopy x C.PyObject_Bytes(x_))) +pybytes(x) = @pyregion pynew(errcheck(@autopy x C.PyObject_Bytes(x_))) pybytes(x::Vector{UInt8}) = pybytes_fromdata(x) pybytes(x::Base.CodeUnits{UInt8,String}) = pybytes_fromdata(x) pybytes(x::Base.CodeUnits{UInt8,SubString{String}}) = pybytes_fromdata(x) @@ -674,7 +686,7 @@ function pyint(x::Unsigned) end end end -pyint(x) = @autopy x pynew(errcheck(C.PyNumber_Long(x_))) +pyint(x) = @pyregion @autopy x pynew(errcheck(C.PyNumber_Long(x_))) pyisint(x) = pytypecheckfast(x, C.Py_TPFLAGS_LONG_SUBCLASS) @@ -686,11 +698,11 @@ pyisint(x) = pytypecheckfast(x, C.Py_TPFLAGS_LONG_SUBCLASS) Convert `x` to a Python `float`. """ pyfloat(x::Real = 0.0) = @pyregion pynew(errcheck(C.PyFloat_FromDouble(x))) -pyfloat(x) = @autopy x pynew(errcheck(C.PyNumber_Float(x_))) +pyfloat(x) = @pyregion @autopy x pynew(errcheck(C.PyNumber_Float(x_))) pyisfloat(x) = pytypecheck(x, pybuiltins.float) -pyfloat_asdouble(x) = errcheck_ambig(@autopy x C.PyFloat_AsDouble(x_)) +pyfloat_asdouble(x) = @pyregion errcheck_ambig(@autopy x C.PyFloat_AsDouble(x_)) ### complex @@ -709,9 +721,11 @@ pycomplex(x, y) = pybuiltins.complex(x, y) pyiscomplex(x) = pytypecheck(x, pybuiltins.complex) function pycomplex_ascomplex(x) - c = @autopy x C.PyComplex_AsCComplex(x_) - c.real == -1 && c.imag == 0 && errcheck() - return Complex(c.real, c.imag) + @pyregion begin + c = @autopy x C.PyComplex_AsCComplex(x_) + c.real == -1 && c.imag == 0 && errcheck() + return Complex(c.real, c.imag) + end end ### type @@ -721,7 +735,7 @@ end The Python `type` of `x`. """ -pytype(x) = pynew(errcheck(@autopy x C.PyObject_Type(x_))) +pytype(x) = @pyregion pynew(errcheck(@autopy x C.PyObject_Type(x_))) """ pytype(name, bases, dict) @@ -798,8 +812,8 @@ end pyistype(x) = pytypecheckfast(x, C.Py_TPFLAGS_TYPE_SUBCLASS) -pytypecheck(x, t) = (@autopy x t C.Py_TypeCheck(x_, t_)) == 1 -pytypecheckfast(x, f) = (@autopy x C.Py_TypeCheckFast(x_, f)) == 1 +pytypecheck(x, t) = @pyregion (@autopy x t C.Py_TypeCheck(x_, t_)) == 1 +pytypecheckfast(x, f) = @pyregion (@autopy x C.Py_TypeCheckFast(x_, f)) == 1 ### slice @@ -809,7 +823,7 @@ pytypecheckfast(x, f) = (@autopy x C.Py_TypeCheckFast(x_, f)) == 1 Construct a Python `slice`. Unspecified arguments default to `None`. """ pyslice(x, y, z = pybuiltins.None) = - pynew(errcheck(@autopy x y z C.PySlice_New(x_, y_, z_))) + @pyregion pynew(errcheck(@autopy x y z C.PySlice_New(x_, y_, z_))) pyslice(y) = pyslice(pybuiltins.None, y, pybuiltins.None) pyisslice(x) = pytypecheck(x, pybuiltins.slice) @@ -893,9 +907,9 @@ function pylist_setitem(xs::Py, i, x) return xs end -pylist_append(xs::Py, x) = errcheck(@autopy x C.PyList_Append(xs, x_)) +pylist_append(xs::Py, x) = @pyregion errcheck(@autopy x C.PyList_Append(xs, x_)) -pylist_astuple(x) = pynew(errcheck(@autopy x C.PyList_AsTuple(x_))) +pylist_astuple(x) = @pyregion pynew(errcheck(@autopy x C.PyList_AsTuple(x_))) function pylist_fromiter(xs) sz = Base.IteratorSize(typeof(xs)) @@ -965,7 +979,7 @@ end ### set -pyset_add(set::Py, x) = (errcheck(@autopy x C.PySet_Add(set, x_)); set) +pyset_add(set::Py, x) = @pyregion (errcheck(@autopy x C.PySet_Add(set, x_)); set) function pyset_update_fromiter(set::Py, xs) for x in xs @@ -1000,7 +1014,7 @@ pyfrozenset(x) = ispy(x) ? pybuiltins.frozenset(x) : pyfrozenset_fromiter(x) ### dict -pydict_setitem(x::Py, k, v) = errcheck(@autopy k v C.PyDict_SetItem(x, k_, v_)) +pydict_setitem(x::Py, k, v) = @pyregion errcheck(@autopy k v C.PyDict_SetItem(x, k_, v_)) function pydict_fromiter(kvs) ans = pydict() @@ -1065,11 +1079,13 @@ pytime(x::Time) = if iszero(nanosecond(x)) pytime(hour(x), minute(x), second(x), millisecond(x) * 1000 + microsecond(x)) else - errset( - pybuiltins.ValueError, - "cannot create 'datetime.time' with less than microsecond resolution", - ) - pythrow() + @pyregion begin + errset( + pybuiltins.ValueError, + "cannot create 'datetime.time' with less than microsecond resolution", + ) + pythrow() + end end pydatetime( @@ -1483,7 +1499,7 @@ Import a module `m`, or an attribute `k`, or a tuple of attributes. If several arguments are given, return the results of importing each one in a tuple. """ -pyimport(m) = pynew(errcheck(@autopy m C.PyImport_Import(m_))) +pyimport(m) = @pyregion pynew(errcheck(@autopy m C.PyImport_Import(m_))) pyimport((m, k)::Pair) = (m_ = pyimport(m); k_ = pygetattr(m_, k); unsafe_pydel(m_); k_) pyimport((m, ks)::Pair{<:Any,<:Tuple}) = (m_ = pyimport(m); ks_ = map(k -> pygetattr(m_, k), ks); unsafe_pydel(m_); ks_) diff --git a/src/Core/err.jl b/src/Core/err.jl index 8e49ab44..5a83dd06 100644 --- a/src/Core/err.jl +++ b/src/Core/err.jl @@ -3,7 +3,7 @@ errval(::T) where {T<:Number} = zero(T) - one(T) iserrval(val) = val == errval(val) -iserrset() = @pyregion C.PyErr_Occurred() != C.PyNULL +iserrset() = C.PyErr_Occurred() != C.PyNULL iserrset(val) = val == errval(val) errcheck() = iserrset() ? pythrow() : nothing @@ -13,35 +13,31 @@ iserrset_ambig(val) = iserrset(val) && iserrset() errcheck_ambig(val) = iserrset_ambig(val) ? pythrow() : val -errclear() = @pyregion C.PyErr_Clear() +errclear() = C.PyErr_Clear() -errmatches(t) = (@autopy t C.PyErr_ExceptionMatches(t_)) == 1 +errmatches(t) = C.PyErr_ExceptionMatches(Py(t)) == 1 function errget() - @pyregion begin - t = Ref(C.PyNULL) - v = Ref(C.PyNULL) - b = Ref(C.PyNULL) - C.PyErr_Fetch(t, v, b) - (pynew(t[]), pynew(v[]), pynew(b[])) - end + t = Ref(C.PyNULL) + v = Ref(C.PyNULL) + b = Ref(C.PyNULL) + C.PyErr_Fetch(t, v, b) + (pynew(t[]), pynew(v[]), pynew(b[])) end -errset(t::Py) = @pyregion C.PyErr_SetNone(t) -errset(t::Py, v::Py) = @pyregion C.PyErr_SetObject(t, v) -errset(t::Py, v::String) = @pyregion C.PyErr_SetString(t, v) +errset(t::Py) = C.PyErr_SetNone(t) +errset(t::Py, v::Py) = C.PyErr_SetObject(t, v) +errset(t::Py, v::String) = C.PyErr_SetString(t, v) function errnormalize!(t::Py, v::Py, b::Py) - @pyregion begin - tref = Ref(getptr(t)) - vref = Ref(getptr(v)) - bref = Ref(getptr(b)) - C.PyErr_NormalizeException(tref, vref, bref) - setptr!(t, tref[]) - setptr!(v, vref[]) - setptr!(b, bref[]) - (t, v, b) - end + tref = Ref(getptr(t)) + vref = Ref(getptr(v)) + bref = Ref(getptr(b)) + C.PyErr_NormalizeException(tref, vref, bref) + setptr!(t, tref[]) + setptr!(v, vref[]) + setptr!(b, bref[]) + (t, v, b) end function PyException(v::Py = pybuiltins.None) @@ -67,15 +63,17 @@ function Base.show(io::IO, x::PyException) end function Base.getproperty(exc::PyException, k::Symbol) - if k in (:t, :v, :b) && !exc._isnormalized - errnormalize!(exc._t, exc._v, exc._b) - pyisnull(exc._t) && pycopy!(exc._t, pybuiltins.None) - pyisnull(exc._v) && pycopy!(exc._v, pybuiltins.None) - pyisnull(exc._b) && pycopy!(exc._b, pybuiltins.None) - pyisnone(exc._v) || (exc._v.__traceback__ = exc._b) - exc._isnormalized = true + @pyregion begin + if k in (:t, :v, :b) && !exc._isnormalized + errnormalize!(exc._t, exc._v, exc._b) + pyisnull(exc._t) && pycopy!(exc._t, pybuiltins.None) + pyisnull(exc._v) && pycopy!(exc._v, pybuiltins.None) + pyisnull(exc._b) && pycopy!(exc._b, pybuiltins.None) + pyisnone(exc._v) || (exc._v.__traceback__ = exc._b) + exc._isnormalized = true + end + k == :t ? exc._t : k == :v ? exc._v : k == :b ? exc._b : getfield(exc, k) end - k == :t ? exc._t : k == :v ? exc._v : k == :b ? exc._b : getfield(exc, k) end pythrow() = throw(PyException(errget()..., false)) diff --git a/src/JlWrap/any.jl b/src/JlWrap/any.jl index e58c2aa6..d65cb597 100644 --- a/src/JlWrap/any.jl +++ b/src/JlWrap/any.jl @@ -298,7 +298,7 @@ function pyjlany_index(self) if self isa Integer pyint(self) else - errset( + @pyregion errset( pybuiltins.TypeError, "Only Julia 'Integer' values can be used as Python indices, not '$(typeof(self))'", ) @@ -310,7 +310,7 @@ function pyjlany_bool(self) if self isa Bool pybool(self) else - errset( + @pyregion errset( pybuiltins.TypeError, "Only Julia 'Bool' values can be tested for truthyness, not '$(typeof(self))'", ) @@ -370,7 +370,7 @@ end function pyjlany_next(self) s = iterate(self) if s === nothing - errset(pybuiltins.StopIteration) + @pyregion errset(pybuiltins.StopIteration) PyNULL else pyjl(s[1]) @@ -380,7 +380,7 @@ end function pyjliter_next(self) s = iterate(self) if s === nothing - errset(pybuiltins.StopIteration) + @pyregion errset(pybuiltins.StopIteration) PyNULL else Py(s[1]) @@ -427,7 +427,7 @@ function pyjlany_numpy_dtype(self::Type) ) end if pyisnull(ans) - errset(pybuiltins.AttributeError, "__numpy_dtype__") + @pyregion errset(pybuiltins.AttributeError, "__numpy_dtype__") end return ans end diff --git a/src/JlWrap/array.jl b/src/JlWrap/array.jl index ebe4a272..dc39da39 100644 --- a/src/JlWrap/array.jl +++ b/src/JlWrap/array.jl @@ -3,16 +3,16 @@ const pyjlarraytype = pynew() function pyjl_getaxisindex(x::AbstractUnitRange{<:Integer}, k::Py) if pyisslice(k) a = @pyconvert Union{Int,Nothing} k.start begin - errset(pybuiltins.TypeError, "slice components must be integers") - pythrow() + @pyregion errset(pybuiltins.TypeError, "slice components must be integers") + @pyregion pythrow() end b = @pyconvert Union{Int,Nothing} k.step begin - errset(pybuiltins.TypeError, "slice components must be integers") - pythrow() + @pyregion errset(pybuiltins.TypeError, "slice components must be integers") + @pyregion pythrow() end c = @pyconvert Union{Int,Nothing} k.stop begin - errset(pybuiltins.TypeError, "slice components must be integers") - pythrow() + @pyregion errset(pybuiltins.TypeError, "slice components must be integers") + @pyregion pythrow() end # step defaults to 1 b′ = b === nothing ? 1 : b @@ -26,8 +26,8 @@ function pyjl_getaxisindex(x::AbstractUnitRange{<:Integer}, k::Py) a′ = Int(last(x)) c′ = Int(first(x)) else - errset(pybuiltins.ValueError, "step must be non-zero") - pythrow() + @pyregion errset(pybuiltins.ValueError, "step must be non-zero") + @pyregion pythrow() end else # start defaults @@ -41,23 +41,23 @@ function pyjl_getaxisindex(x::AbstractUnitRange{<:Integer}, k::Py) if checkbounds(Bool, x, r) return r else - errset(pybuiltins.IndexError, "array index out of bounds") - pythrow() + @pyregion errset(pybuiltins.IndexError, "array index out of bounds") + @pyregion pythrow() end else j = @pyconvert Int k begin - errset( + @pyregion errset( pybuiltins.TypeError, "index must be slice or integer, got '$(pytype(k).__name__)'", ) - pythrow() + @pyregion pythrow() end r = Int(j < 0 ? (last(x) + j + 1) : (first(x) + j)) if checkbounds(Bool, x, r) return r else - errset(pybuiltins.IndexError, "array index out of bounds") - pythrow() + @pyregion errset(pybuiltins.IndexError, "array index out of bounds") + @pyregion pythrow() end end end @@ -72,13 +72,13 @@ function pyjl_getarrayindices(x::AbstractArray{T,N}, ks::Py) where {T,N} return ans end else - errset(pybuiltins.TypeError, "expecting $N indices, got $(pylen(ks))") - pythrow() + @pyregion errset(pybuiltins.TypeError, "expecting $N indices, got $(pylen(ks))") + @pyregion pythrow() end elseif N == 1 return (pyjl_getaxisindex(axes(x, 1), ks),) else - errset(pybuiltins.TypeError, "expecting $N indices, got 1") + @pyregion errset(pybuiltins.TypeError, "expecting $N indices, got 1") end end @@ -111,8 +111,8 @@ function pyjlarray_delitem(x::AbstractArray{T,N}, k_::Py) where {T,N} unsafe_pydel(k_) deleteat!(x, k...) else - errset(pybuiltins.TypeError, "can only delete from 1D arrays") - pythrow() + @pyregion errset(pybuiltins.TypeError, "can only delete from 1D arrays") + @pyregion pythrow() end return Py(nothing) end @@ -320,7 +320,7 @@ function pyjlarray_array_interface(x::AbstractArray{T,N}) where {T,N} return d end end - errset(pybuiltins.AttributeError, "__array_interface__") + @pyregion errset(pybuiltins.AttributeError, "__array_interface__") return PyNULL end pyjl_handle_error_type(::typeof(pyjlarray_array_interface), x, exc) = diff --git a/src/JlWrap/base.jl b/src/JlWrap/base.jl index 67723e59..29c8ec36 100644 --- a/src/JlWrap/base.jl +++ b/src/JlWrap/base.jl @@ -4,7 +4,7 @@ _pyjl_getvalue(x) = @autopy x Cjl.PyJuliaValue_GetValue(x_) _pyjl_setvalue!(x, v) = @autopy x Cjl.PyJuliaValue_SetValue(x_, v) -pyjl(t, v) = pynew(errcheck(@autopy t Cjl.PyJuliaValue_New(t_, v))) +pyjl(t, v) = @pyregion pynew(errcheck(@autopy t Cjl.PyJuliaValue_New(t_, v))) """ pyisjl(x) @@ -34,57 +34,59 @@ pyconvert_rule_jlvalue(::Type{T}, x::Py) where {T} = function Cjl._pyjl_callmethod(f, self_::C.PyPtr, args_::C.PyPtr, nargs::C.Py_ssize_t) @nospecialize f - in_f = false - self = Cjl.PyJuliaValue_GetValue(self_) - try - if nargs == 1 - in_f = true - ans = @pyregionbreak(f(self))::Py - in_f = false - elseif nargs == 2 - arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) - in_f = true - ans = @pyregionbreak(f(self, arg1))::Py - in_f = false - elseif nargs == 3 - arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) - arg2 = pynew(incref(C.PyTuple_GetItem(args_, 2))) - in_f = true - ans = @pyregionbreak(f(self, arg1, arg2))::Py - in_f = false - elseif nargs == 4 - arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) - arg2 = pynew(incref(C.PyTuple_GetItem(args_, 2))) - arg3 = pynew(incref(C.PyTuple_GetItem(args_, 3))) - in_f = true - ans = @pyregionbreak(f(self, arg1, arg2, arg3))::Py - in_f = false - else - errset( - pybuiltins.NotImplementedError, - "__jl_callmethod not implemented for this many arguments", - ) - end - return getptr(incref(ans)) - catch exc - if exc isa PyException - Base.GC.@preserve exc C.PyErr_Restore( - incref(exc._t), - incref(exc._v), - incref(exc._b), - ) - return C.PyNULL - else - try - if in_f - return pyjl_handle_error(f, self, exc) - else - errset(pyJuliaError, pytuple((pyjl(exc), pyjl(catch_backtrace())))) + @pyregion begin + in_f = false + self = Cjl.PyJuliaValue_GetValue(self_) + try + if nargs == 1 + in_f = true + ans = @pyregionbreak(f(self))::Py + in_f = false + elseif nargs == 2 + arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) + in_f = true + ans = @pyregionbreak(f(self, arg1))::Py + in_f = false + elseif nargs == 3 + arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) + arg2 = pynew(incref(C.PyTuple_GetItem(args_, 2))) + in_f = true + ans = @pyregionbreak(f(self, arg1, arg2))::Py + in_f = false + elseif nargs == 4 + arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) + arg2 = pynew(incref(C.PyTuple_GetItem(args_, 2))) + arg3 = pynew(incref(C.PyTuple_GetItem(args_, 3))) + in_f = true + ans = @pyregionbreak(f(self, arg1, arg2, arg3))::Py + in_f = false + else + @pyregion errset( + pybuiltins.NotImplementedError, + "__jl_callmethod not implemented for this many arguments", + ) + end + return getptr(incref(ans)) + catch exc + if exc isa PyException + Base.GC.@preserve exc C.PyErr_Restore( + incref(exc._t), + incref(exc._v), + incref(exc._b), + ) + return C.PyNULL + else + try + if in_f + return pyjl_handle_error(f, self, exc) + else + @pyregion errset(pyJuliaError, pytuple((pyjl(exc), pyjl(catch_backtrace())))) + return C.PyNULL + end + catch + @pyregion errset(pyJuliaError, "an error occurred while setting an error") return C.PyNULL end - catch - errset(pyJuliaError, "an error occurred while setting an error") - return C.PyNULL end end end @@ -95,11 +97,11 @@ function pyjl_handle_error(f, self, exc) t = pyjl_handle_error_type(f, self, exc)::Py if pyisnull(t) # NULL => raise JuliaError - errset(pyJuliaError, pytuple((pyjl(exc), pyjl(catch_backtrace())))) + @pyregion errset(pyJuliaError, pytuple((pyjl(exc), pyjl(catch_backtrace())))) return C.PyNULL elseif pyistype(t) # Exception type => raise this type of error - errset(t, string("Julia: ", Py(sprint(showerror, exc)))) + @pyregion errset(t, string("Julia: ", Py(sprint(showerror, exc)))) return C.PyNULL else # Otherwise, return the given object (e.g. NotImplemented) diff --git a/src/JlWrap/io.jl b/src/JlWrap/io.jl index 0f7c07d6..ed47400f 100644 --- a/src/JlWrap/io.jl +++ b/src/JlWrap/io.jl @@ -35,7 +35,7 @@ function pyjlio_seek(io::IO, offset_::Py, whence_::Py) seekend(io) pos = position(io) + offset else - errset(pybuiltins.ValueError, "Argument 'whence' must be 0, 1 or 2") + @pyregion errset(pybuiltins.ValueError, "Argument 'whence' must be 0, 1 or 2") return PyNULL end seek(io, pos) @@ -98,14 +98,14 @@ function pyjlbinaryio_readinto(io::IO, b::Py) c = m.c_contiguous if !pytruth(c) unsafe_pydel(c) - errset(pybuiltins.ValueError, "input buffer is not contiguous") + @pyregion errset(pybuiltins.ValueError, "input buffer is not contiguous") return PyNULL end unsafe_pydel(c) buf = unsafe_load(C.PyMemoryView_GET_BUFFER(m)) if buf.readonly != 0 unsafe_pydel(m) - errset(pybuiltins.ValueError, "output buffer is read-only") + @pyregion errset(pybuiltins.ValueError, "output buffer is read-only") return PyNULL end data = unsafe_wrap(Array, Ptr{UInt8}(buf.buf), buf.len) @@ -121,7 +121,7 @@ function pyjlbinaryio_write(io::IO, b::Py) c = m.c_contiguous if !pytruth(c) unsafe_pydel(c) - errset(pybuiltins.ValueError, "input buffer is not contiguous") + @pyregion errset(pybuiltins.ValueError, "input buffer is not contiguous") return PyNULL end unsafe_pydel(c) @@ -212,7 +212,7 @@ function pyjltextio_write(io::IO, s_::Py) # TODO: is this the number of source characters, or the number of output characters? Py(length(s)) else - errset( + @pyregion errset( pybuiltins.TypeError, "Argument 's' must be a 'str', got a '$(pytype(s_).__name__)'", ) diff --git a/src/JlWrap/set.jl b/src/JlWrap/set.jl index 36f19f02..9bf5e3f8 100644 --- a/src/JlWrap/set.jl +++ b/src/JlWrap/set.jl @@ -11,7 +11,7 @@ end function pyjlset_pop(x::AbstractSet) if isempty(x) - errset(pybuiltins.KeyError, "pop from an empty set") + @pyregion errset(pybuiltins.KeyError, "pop from an empty set") PyNULL else Py(pop!(x)) @@ -20,14 +20,14 @@ end function pyjlset_remove(x::AbstractSet, v_::Py) v = @pyconvert eltype(x) v_ begin - errset(pybuiltins.KeyError, v_) + @pyregion errset(pybuiltins.KeyError, v_) return PyNULL end if v in x delete!(x, v) return Py(nothing) else - errset(pybuiltins.KeyError, v_) + @pyregion errset(pybuiltins.KeyError, v_) return PyNULL end end diff --git a/src/JlWrap/vector.jl b/src/JlWrap/vector.jl index 8058098c..0e39effc 100644 --- a/src/JlWrap/vector.jl +++ b/src/JlWrap/vector.jl @@ -39,7 +39,7 @@ function pyjlvector_insert(x::AbstractVector, k_::Py, v_::Py) insert!(x, k′, v) return Py(nothing) else - errset(pybuiltins.IndexError, "array index out of bounds") + @pyregion errset(pybuiltins.IndexError, "array index out of bounds") return PyNULL end end @@ -75,19 +75,19 @@ function pyjlvector_pop(x::AbstractVector, k_::Py) end return Py(v) else - errset(pybuiltins.IndexError, "pop from empty array") + @pyregion errset(pybuiltins.IndexError, "pop from empty array") return PyNULL end end function pyjlvector_remove(x::AbstractVector, v_::Py) v = @pyconvert eltype(x) v_ begin - errset(pybuiltins.ValueError, "value not in array") + @pyregion errset(pybuiltins.ValueError, "value not in array") return PyNULL end k = findfirst(==(v), x) if k === nothing - errset(pybuiltins.ValueError, "value not in array") + @pyregion errset(pybuiltins.ValueError, "value not in array") return PyNULL end deleteat!(x, k) @@ -96,12 +96,12 @@ end function pyjlvector_index(x::AbstractVector, v_::Py) v = @pyconvert eltype(x) v_ begin - errset(pybuiltins.ValueError, "value not in array") + @pyregion errset(pybuiltins.ValueError, "value not in array") return PyNULL end k = findfirst(==(v), x) if k === nothing - errset(pybuiltins.ValueError, "value not in array") + @pyregion errset(pybuiltins.ValueError, "value not in array") return PyNULL end Py(k - first(axes(x, 1)))