From 3369142a7d5f2fd294a14fe45d7bc8baba2d00ef Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Sat, 19 Sep 2026 19:47:05 +0100 Subject: [PATCH 1/7] Manage Python thread states per Julia task --- AGENTS.md | 5 +- Project.toml | 2 +- docs/src/juliacall.md | 23 +-- docs/src/pythoncall-reference.md | 11 +- docs/src/pythoncall.md | 35 ++--- pysrc/juliacall/juliapkg-dev.json | 2 +- pysrc/juliacall/juliapkg.json | 2 +- src/API/exports.jl | 2 + src/API/macros.jl | 2 + src/C/context.jl | 9 +- src/C/pointers.jl | 9 ++ src/Core/Core.jl | 2 + src/Core/Py.jl | 26 ++- src/Core/builtins.jl | 78 +++++---- src/Core/err.jl | 40 ++--- src/GC/GC.jl | 21 ++- src/GIL/GIL.jl | 47 +++--- src/JlWrap/any.jl | 13 +- src/JlWrap/base.jl | 8 +- src/PyMacro/PyMacro.jl | 18 ++- src/PythonCall.jl | 5 + src/Region/Region.jl | 253 ++++++++++++++++++++++++++++++ test/GC.jl | 4 + test/Region.jl | 42 +++++ 24 files changed, 493 insertions(+), 166 deletions(-) create mode 100644 src/Region/Region.jl create mode 100644 test/Region.jl diff --git a/AGENTS.md b/AGENTS.md index 354305bd..236a5279 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,7 +5,10 @@ - **Python tests**: - Copy `pysrc/juliacall/juliapkg-dev.json` to `pysrc/juliacall/juliapkg.json` before running (do **not** commit this copy). - Execute with `uv run pytest -s --nbval ./pytest` (add `--cov=pysrc` when coverage is needed). - - Sometimes `juliapkg` requires Julia 1.10–1.11; `juliaup` already provides 1.11.7 in this environment. +- Sometimes `juliapkg` requires Julia 1.10–1.11; `juliaup` already provides 1.11.7 in this environment. +- With Python 3.14, juliapkg's OpenSSL compatibility currently constrains Julia to 1.11 or + older. Packages requiring Julia 1.12+ therefore cannot run the Python suite under that + Python; use an older Python whose OpenSSL constraint permits Julia 1.12+. The majority of tests live in the Julia package; Python tests cover functionality that cannot be exercised from Julia (e.g., JuliaCall-specific behavior). Run both suites—typically Julia first—in whichever order makes sense. diff --git a/Project.toml b/Project.toml index 9e6d15fe..970b2162 100644 --- a/Project.toml +++ b/Project.toml @@ -34,7 +34,7 @@ PyCall = "1" Serialization = "1" Tables = "1" UnsafePointers = "1" -julia = "1.10" +julia = "1.12" [extras] CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" diff --git a/docs/src/juliacall.md b/docs/src/juliacall.md index b6c073b0..8deb7f07 100644 --- a/docs/src/juliacall.md +++ b/docs/src/juliacall.md @@ -157,13 +157,11 @@ caveats. Most importantly, you can only call Python code while Python's [Global Interpreter Lock (GIL)](https://docs.python.org/3/glossary.html#term-global-interpreter-lock) -is locked by the current thread. You can use JuliaCall from any Python thread, and the GIL -will be locked whenever any JuliaCall function is used. However, to leverage the benefits -of multi-threading, you can unlock the GIL while executing any Julia code that does not -interact with Python. +JuliaCall borrows the Python thread state which entered Julia and automatically detaches it +while arbitrary Julia code runs. Nested Python interaction from that Julia code temporarily +reattaches the same state, and the borrowed state is restored before returning to Python. -The simplest way to do this is using the `_jl_call_nogil` method on Julia functions to -call the function with the GIL unlocked. +The historical `_jl_call_nogil` spelling remains available as a compatibility alias: ```python from concurrent.futures import ThreadPoolExecutor, wait @@ -173,16 +171,9 @@ fs = [pool.submit(jl.Libc.systemsleep._jl_call_nogil, 5) for _ in range(4)] wait(fs) ``` -In the above example, we call `Libc.systemsleep(5)` on four threads. Because we -called it with `_jl_call_nogil`, the GIL was unlocked, allowing the threads to run in -parallel, taking about 5 seconds in total. - -If we did not use `_jl_call_nogil` (i.e. if we did `pool.submit(jl.Libc.systemsleep, 5)`) -then the above code will take 20 seconds because the sleeps run one after another. - -It is very important that any function called with `_jl_call_nogil` does not interact -with Python at all unless it re-locks the GIL first, such as by using -[PythonCall.GIL.@lock](@ref). +Ordinary calls provide the same automatic resource management, so +`pool.submit(jl.Libc.systemsleep, 5)` is preferred. PythonCall operations nested inside +Julia callbacks are safe without explicit region or lock calls. You can also use [multi-threading from Julia](@ref jl-multi-threading). diff --git a/docs/src/pythoncall-reference.md b/docs/src/pythoncall-reference.md index 6c13d3fb..9b6400f5 100644 --- a/docs/src/pythoncall-reference.md +++ b/docs/src/pythoncall-reference.md @@ -222,8 +222,15 @@ Py(x::MyType) = x.py ## Multi-threading -These functions are not exported. They support multi-threading of Python and/or Julia. -See also [`juliacall.AnyValue._jl_call_nogil`](@ref julia-wrappers). +PythonCall manages Python thread state automatically. These exported macros are optional +performance and concurrency hints; users normally do not need them for correctness. + +```@docs +@pyregion +@pyregionbreak +``` + +The older `PythonCall.GIL` names remain as compatibility aliases. ```@docs PythonCall.GIL.lock diff --git a/docs/src/pythoncall.md b/docs/src/pythoncall.md index 9bdc42a2..3659329f 100644 --- a/docs/src/pythoncall.md +++ b/docs/src/pythoncall.md @@ -471,22 +471,15 @@ See [Installing Python packages](@ref python-deps). Multi-threading support is experimental and can change without notice. -From v0.9.22, PythonCall supports multi-threading in Julia and/or Python, with some -caveats. - -Most importantly, you can only call Python code while Python's -[Global Interpreter Lock (GIL)](https://docs.python.org/3/glossary.html#term-global-interpreter-lock) -is locked by the current thread. Ordinarily, the GIL is locked by the main thread in Julia, -so if you want to run Python code on any other thread, you must unlock the GIL from the -main thread and then re-lock it while running any Python code on other threads. - -This is made possible by the macros [`PythonCall.GIL.@unlock`](@ref) and -[`PythonCall.GIL.@lock`](@ref) or the functions [`PythonCall.GIL.unlock`](@ref) and -[`PythonCall.GIL.lock`](@ref) with this pattern: +PythonCall APIs automatically establish the Python thread state they need, so ordinary +operations can be called from any Julia task or thread without explicit locking. The +optional [`@pyregion`](@ref) macro amortizes those transitions across straight-line, +Python-heavy work. Use [`@pyregionbreak`](@ref) around Julia-heavy code which deliberately +yields, waits, or blocks cooperatively: ```julia -PythonCall.GIL.@unlock Threads.@threads for i in 1:4 - PythonCall.GIL.@lock pyimport("time").sleep(5) +Threads.@threads for i in 1:4 + @pyregion pyimport("time").sleep(5) end ``` @@ -494,9 +487,10 @@ In the above example, we call `time.sleep(5)` four times in parallel. If Julia w started with at least four threads (`julia -t4`) then the above code will take about 5 seconds. -Both `@unlock` and `@lock` are important. If the GIL were not unlocked, then a deadlock -would occur when attempting to lock the already-locked GIL from the threads. If the GIL -were not re-locked, then Python would crash when interacting with it. +Both region macros nest arbitrarily, and neither is required for correctness. A nested +PythonCall operation inside `@pyregionbreak` temporarily re-enters Python automatically. +On a GIL-enabled Python, attaching a state can block that Julia worker while CPython +arbitrates access; free-threaded Python uses the same state-management machinery. With multiple Julia threads you need exactly one interactive thread, see the [FAQ](@ref faq-multi-threading). @@ -504,9 +498,8 @@ You can also use [multi-threading from Python](@ref py-multi-threading). ### Caveat: Garbage collection -If Julia's GC collects any Python objects from a thread where the GIL is not currently -locked, then those Python objects will not immediately be deleted. Instead they will be -queued to be deleted in a later GC pass. +If Julia's GC collects Python objects while no Python thread state is already attached, +those objects are queued rather than making a finalizer block while attaching a state. If you find you have many Python objects not being deleted, you can call -[`PythonCall.GC.gc()`](@ref) or `GC.gc()` while the GIL is locked to clear the queue. +[`PythonCall.GC.gc()`](@ref) or `GC.gc()` to clear the queue. diff --git a/pysrc/juliacall/juliapkg-dev.json b/pysrc/juliacall/juliapkg-dev.json index 3e6910a8..3dd2bbd1 100644 --- a/pysrc/juliacall/juliapkg-dev.json +++ b/pysrc/juliacall/juliapkg-dev.json @@ -1,5 +1,5 @@ { - "julia": "^1.10.3", + "julia": "^1.12", "packages": { "PythonCall": { "uuid": "6099a3de-0909-46bc-b1f4-468b9a2dfc0d", diff --git a/pysrc/juliacall/juliapkg.json b/pysrc/juliacall/juliapkg.json index f45ce3b1..37f68861 100644 --- a/pysrc/juliacall/juliapkg.json +++ b/pysrc/juliacall/juliapkg.json @@ -1,5 +1,5 @@ { - "julia": "^1.10.3", + "julia": "^1.12", "packages": { "PythonCall": { "uuid": "6099a3de-0909-46bc-b1f4-468b9a2dfc0d", diff --git a/src/API/exports.jl b/src/API/exports.jl index cdc5ac83..d2a048fc 100644 --- a/src/API/exports.jl +++ b/src/API/exports.jl @@ -3,6 +3,8 @@ export @py export @pyconst export @pyeval export @pyexec +export @pyregion +export @pyregionbreak export ispy export Py export pyabs diff --git a/src/API/macros.jl b/src/API/macros.jl index c74d191c..741ae191 100644 --- a/src/API/macros.jl +++ b/src/API/macros.jl @@ -2,6 +2,8 @@ macro pyconst end macro pyeval end macro pyexec end +macro pyregion end +macro pyregionbreak end # Convert macro pyconvert end diff --git a/src/C/context.jl b/src/C/context.jl index 6ccdab89..220bf88d 100644 --- a/src/C/context.jl +++ b/src/C/context.jl @@ -21,6 +21,11 @@ A handle to a loaded instance of libpython, its interpreter, function pointers, end const CTX = Context() +const FINALIZE_HOOK = Ref{Function}(() -> begin + if Py_FinalizeEx() == -1 + @warn "Py_FinalizeEx() error" + end +end) function _atpyexit() if CTX.is_initialized && !CTX.is_preinitialized @@ -282,9 +287,7 @@ function init_context() Py_InitializeEx(0) atexit() do CTX.is_initialized = false - if Py_FinalizeEx() == -1 - @warn "Py_FinalizeEx() error" - end + FINALIZE_HOOK[]() end end CTX.is_initialized = true diff --git a/src/C/pointers.jl b/src/C/pointers.jl index 9644329f..6108fa3b 100644 --- a/src/C/pointers.jl +++ b/src/C/pointers.jl @@ -19,6 +19,8 @@ const CAPI_FUNC_SIGS = Dict{Symbol,Pair{Tuple,Type}}( # GIL & THREADS :PyEval_SaveThread => () => Ptr{Cvoid}, :PyEval_RestoreThread => (Ptr{Cvoid},) => Cvoid, + :PyThreadState_New => (Ptr{Cvoid},) => Ptr{Cvoid}, + :PyThreadState_GetInterpreter => (Ptr{Cvoid},) => Ptr{Cvoid}, :PyGILState_Ensure => () => PyGILState_STATE, :PyGILState_Release => (PyGILState_STATE,) => Cvoid, :PyGILState_GetThisThreadState => () => Ptr{Cvoid}, @@ -278,6 +280,7 @@ const CAPI_OBJECTS = Set([ $([:($name::PyPtr = C_NULL) for name in CAPI_EXCEPTIONS]...) $([:($name::PyPtr = C_NULL) for name in CAPI_OBJECTS]...) PyOS_InputHookPtr::Ptr{Ptr{Cvoid}} = C_NULL + PyThreadState_GetUnchecked::Ptr{Cvoid} = C_NULL end const POINTERS = CAPIPointers() @@ -295,8 +298,14 @@ const POINTERS = CAPIPointers() ) $([:(p.$name = dlsym(lib, $(QuoteNode(name)))) for name in CAPI_OBJECTS]...) p.PyOS_InputHookPtr = dlsym(CTX.lib_ptr, :PyOS_InputHook) + p.PyThreadState_GetUnchecked = let q = dlsym_e(lib, :PyThreadState_GetUnchecked) + q == C_NULL ? dlsym(lib, :_PyThreadState_UncheckedGet) : q + end end +PyThreadState_GetUnchecked() = + ccall(POINTERS.PyThreadState_GetUnchecked, Ptr{Cvoid}, ()) + for (name, (argtypes, rettype)) in CAPI_FUNC_SIGS args = [Symbol("x", i) for (i, _) in enumerate(argtypes)] @eval $name($(args...)) = ccall(POINTERS.$name, $rettype, ($(argtypes...),), $(args...)) diff --git a/src/Core/Core.jl b/src/Core/Core.jl index 628643b6..ebb0becf 100644 --- a/src/Core/Core.jl +++ b/src/Core/Core.jl @@ -32,6 +32,8 @@ using Markdown: Markdown import ..PythonCall: @pyconst, + @pyregion, + @pyregionbreak, @pyeval, @pyexec, ispy, diff --git a/src/Core/Py.jl b/src/Core/Py.jl index 50ff0eb2..f86a1670 100644 --- a/src/Core/Py.jl +++ b/src/Core/Py.jl @@ -1,5 +1,5 @@ -incref(x::C.PyPtr) = (C.Py_IncRef(x); x) -decref(x::C.PyPtr) = (C.Py_DecRef(x); x) +incref(x::C.PyPtr) = @pyregion (C.Py_IncRef(x); x) +decref(x::C.PyPtr) = @pyregion (C.Py_DecRef(x); x) """ ispy(x) @@ -85,10 +85,12 @@ Use this to eagerly free a Python object, rather than waiting for Julia's GC to it at some indeterminate point in the future. """ function unsafe_pydel(x::Py) - ptr = getptr(x) - if ptr != C.PyNULL - C.Py_DecRef(ptr) - setptr!(x, C.PyNULL) + @pyregion begin + ptr = getptr(x) + if ptr != C.PyNULL + C.Py_DecRef(ptr) + setptr!(x, C.PyNULL) + end end return end @@ -99,12 +101,14 @@ macro autopy(args...) body = args[end] # ans = gensym("ans") esc(quote + @pyregion begin # $([:($t = $ispy($v) ? $v : $Py($v)) for (t, v) in zip(ts, vs)]...) # $ans = $body # $([:($ispy($v) || $unsafe_pydel($t)) for (t, v) in zip(ts, vs)]...) # $ans $([:($t = $Py($v)) for (t, v) in zip(ts, vs)]...) $body + end end) end @@ -291,15 +295,7 @@ function _propertynames(x::Py, private::Bool) return Symbol[Symbol(pystr_asstring(word)) for word in words] end -function Base.propertynames(x::Py, private::Bool = false) - if C.PyGILState_Check() == 1 - _propertynames(x, private) - else - C.on_main_thread() do - _propertynames(x, private) - end::Vector{Symbol} - end -end +Base.propertynames(x::Py, private::Bool = false) = @pyregion _propertynames(x, private) Base.Bool(x::Py) = pytruth(x) diff --git a/src/Core/builtins.jl b/src/Core/builtins.jl index 1bb634cb..13faca50 100644 --- a/src/Core/builtins.jl +++ b/src/Core/builtins.jl @@ -515,7 +515,7 @@ Returns the next item from the iterator `x`. If there are no more items, returns given, else raises `StopIteration`. """ function pynext(x) - ptr = errcheck_ambig(C.PyIter_Next(x)) + ptr = @pyregion errcheck_ambig(C.PyIter_Next(x)) if ptr == C.PyNULL errset(pybuiltins.StopIteration) pythrow() @@ -525,7 +525,7 @@ function pynext(x) end function pynext(x, d) - ptr = errcheck_ambig(C.PyIter_Next(x)) + ptr = @pyregion errcheck_ambig(C.PyIter_Next(x)) ptr == C.PyNULL ? d : pynew(ptr) end @@ -534,7 +534,8 @@ end Return the next item in the iterator `x`. When there are no more items, return NULL. """ -unsafe_pynext(x::Py) = Base.GC.@preserve x pynew(errcheck_ambig(C.PyIter_Next(x))) +unsafe_pynext(x::Py) = + @pyregion Base.GC.@preserve x pynew(errcheck_ambig(C.PyIter_Next(x))) ### None @@ -567,7 +568,8 @@ end ### str -pystr_fromUTF8(x::Ptr, n::Integer) = pynew(errcheck(C.PyUnicode_DecodeUTF8(x, n, C_NULL))) +pystr_fromUTF8(x::Ptr, n::Integer) = + @pyregion pynew(errcheck(C.PyUnicode_DecodeUTF8(x, n, C_NULL))) pystr_fromUTF8(x) = pystr_fromUTF8(pointer(x), sizeof(x)) """ @@ -583,23 +585,26 @@ pystr(x::AbstractString) = pystr(convert(String, x)::String) pystr(x::AbstractChar) = pystr(convert(Char, x)::Char) pystr(::Type{String}, x) = (s = pystr(x); ans = pystr_asstring(s); unsafe_pydel(s); ans) -pystr_asUTF8bytes(x::Py) = pynew(errcheck(C.PyUnicode_AsUTF8String(x))) +pystr_asUTF8bytes(x::Py) = @pyregion pynew(errcheck(C.PyUnicode_AsUTF8String(x))) pystr_asUTF8vector(x::Py) = (b = pystr_asUTF8bytes(x); ans = pybytes_asvector(b); unsafe_pydel(b); ans) pystr_asstring(x::Py) = (b = pystr_asUTF8bytes(x); ans = pybytes_asUTF8string(b); unsafe_pydel(b); ans) function pystr_intern!(x::Py) - ptr = Ref(getptr(x)) - C.PyUnicode_InternInPlace(ptr) - setptr!(x, ptr[]) + @pyregion begin + ptr = Ref(getptr(x)) + C.PyUnicode_InternInPlace(ptr) + setptr!(x, ptr[]) + end end pyisstr(x) = pytypecheckfast(x, C.Py_TPFLAGS_UNICODE_SUBCLASS) ### bytes -pybytes_fromdata(x::Ptr, n::Integer) = pynew(errcheck(C.PyBytes_FromStringAndSize(x, n))) +pybytes_fromdata(x::Ptr, n::Integer) = + @pyregion pynew(errcheck(C.PyBytes_FromStringAndSize(x, n))) pybytes_fromdata(x) = pybytes_fromdata(pointer(x), sizeof(x)) """ @@ -619,10 +624,12 @@ pybytes(::Type{T}, x) where {Base.CodeUnits{UInt8,String} <: T <: Base.CodeUnits pyisbytes(x) = pytypecheckfast(x, C.Py_TPFLAGS_BYTES_SUBCLASS) function pybytes_asdata(x::Py) - ptr = Ref(Ptr{Cchar}(0)) - len = Ref(C.Py_ssize_t(0)) - errcheck(C.PyBytes_AsStringAndSize(x, ptr, len)) - ptr[], len[] + @pyregion begin + ptr = Ref(Ptr{Cchar}(0)) + len = Ref(C.Py_ssize_t(0)) + errcheck(C.PyBytes_AsStringAndSize(x, ptr, len)) + ptr[], len[] + end end function pybytes_asvector(x::Py) @@ -648,19 +655,23 @@ pyint_fallback(x::Integer) = pyint_fallback(BigInt(x)) Convert `x` to a Python `int`. """ function pyint(x::Integer = 0) - y = mod(x, Clonglong) - if x == y - pynew(errcheck(C.PyLong_FromLongLong(y))) - else - pyint_fallback(x) + @pyregion begin + y = mod(x, Clonglong) + if x == y + pynew(errcheck(C.PyLong_FromLongLong(y))) + else + pyint_fallback(x) + end end end function pyint(x::Unsigned) - y = mod(x, Culonglong) - if x == y - pynew(errcheck(C.PyLong_FromUnsignedLongLong(y))) - else - pyint_fallback(x) + @pyregion begin + y = mod(x, Culonglong) + if x == y + pynew(errcheck(C.PyLong_FromUnsignedLongLong(y))) + else + pyint_fallback(x) + end end end pyint(x) = @autopy x pynew(errcheck(C.PyNumber_Long(x_))) @@ -674,7 +685,7 @@ pyisint(x) = pytypecheckfast(x, C.Py_TPFLAGS_LONG_SUBCLASS) Convert `x` to a Python `float`. """ -pyfloat(x::Real = 0.0) = pynew(errcheck(C.PyFloat_FromDouble(x))) +pyfloat(x::Real = 0.0) = @pyregion pynew(errcheck(C.PyFloat_FromDouble(x))) pyfloat(x) = @autopy x pynew(errcheck(C.PyNumber_Float(x_))) pyisfloat(x) = pytypecheck(x, pybuiltins.float) @@ -689,7 +700,8 @@ pyfloat_asdouble(x) = errcheck_ambig(@autopy x C.PyFloat_AsDouble(x_)) Convert `x` to a Python `complex`, or create one from given real and imaginary parts. """ -pycomplex(x::Real = 0.0, y::Real = 0.0) = pynew(errcheck(C.PyComplex_FromDoubles(x, y))) +pycomplex(x::Real = 0.0, y::Real = 0.0) = + @pyregion pynew(errcheck(C.PyComplex_FromDoubles(x, y))) pycomplex(x::Complex) = pycomplex(real(x), imag(x)) pycomplex(x) = pybuiltins.complex(x) pycomplex(x, y) = pybuiltins.complex(x, y) @@ -819,15 +831,15 @@ pyisrange(x) = pytypecheck(x, pybuiltins.range) ### tuple -pynulltuple(len) = pynew(errcheck(C.PyTuple_New(len))) +pynulltuple(len) = @pyregion pynew(errcheck(C.PyTuple_New(len))) function pytuple_setitem(xs::Py, i, x) - errcheck(C.PyTuple_SetItem(xs, i, incref(Py(x)))) + @pyregion errcheck(C.PyTuple_SetItem(xs, i, incref(Py(x)))) return xs end function pytuple_getitem(xs::Py, i) - Base.GC.@preserve xs pynew(incref(errcheck(C.PyTuple_GetItem(xs, i)))) + @pyregion Base.GC.@preserve xs pynew(incref(errcheck(C.PyTuple_GetItem(xs, i)))) end function pytuple_fromiter(xs) @@ -874,10 +886,10 @@ pyistuple(x) = pytypecheckfast(x, C.Py_TPFLAGS_TUPLE_SUBCLASS) ### list -pynulllist(len) = pynew(errcheck(C.PyList_New(len))) +pynulllist(len) = @pyregion pynew(errcheck(C.PyList_New(len))) function pylist_setitem(xs::Py, i, x) - errcheck(C.PyList_SetItem(xs, i, incref(Py(x)))) + @pyregion errcheck(C.PyList_SetItem(xs, i, incref(Py(x)))) return xs end @@ -972,7 +984,7 @@ Convert `x` to a Python `set`. If `x` is a Python object, this is equivalent to `set(x)` in Python. Otherwise `x` must be iterable. """ -pyset() = pynew(errcheck(C.PySet_New(C.PyNULL))) +pyset() = @pyregion pynew(errcheck(C.PySet_New(C.PyNULL))) pyset(x) = ispy(x) ? pybuiltins.set(x) : pyset_fromiter(x) """ @@ -983,7 +995,7 @@ Convert `x` to a Python `frozenset`. If `x` is a Python object, this is equivalent to `frozenset(x)` in Python. Otherwise `x` must be iterable. """ -pyfrozenset() = pynew(errcheck(C.PyFrozenSet_New(C.PyNULL))) +pyfrozenset() = @pyregion pynew(errcheck(C.PyFrozenSet_New(C.PyNULL))) pyfrozenset(x) = ispy(x) ? pybuiltins.frozenset(x) : pyfrozenset_fromiter(x) ### dict @@ -1017,7 +1029,7 @@ If `x` is a Python object, this is equivalent to `dict(x)` in Python. Otherwise `x` must iterate over key-value pairs. """ pydict(; kwargs...) = - isempty(kwargs) ? pynew(errcheck(C.PyDict_New())) : pystrdict_fromiter(kwargs) + isempty(kwargs) ? (@pyregion pynew(errcheck(C.PyDict_New()))) : pystrdict_fromiter(kwargs) pydict(x) = ispy(x) ? pybuiltins.dict(x) : pydict_fromiter(x) pydict(x::NamedTuple) = pydict(; x...) pydict(pair::Pair, pairs::Pair...) = pydict((pair, pairs...)) diff --git a/src/Core/err.jl b/src/Core/err.jl index 4ee00c74..8e49ab44 100644 --- a/src/Core/err.jl +++ b/src/Core/err.jl @@ -3,7 +3,7 @@ errval(::T) where {T<:Number} = zero(T) - one(T) iserrval(val) = val == errval(val) -iserrset() = C.PyErr_Occurred() != C.PyNULL +iserrset() = @pyregion C.PyErr_Occurred() != C.PyNULL iserrset(val) = val == errval(val) errcheck() = iserrset() ? pythrow() : nothing @@ -13,31 +13,35 @@ iserrset_ambig(val) = iserrset(val) && iserrset() errcheck_ambig(val) = iserrset_ambig(val) ? pythrow() : val -errclear() = C.PyErr_Clear() +errclear() = @pyregion C.PyErr_Clear() errmatches(t) = (@autopy t C.PyErr_ExceptionMatches(t_)) == 1 function errget() - t = Ref(C.PyNULL) - v = Ref(C.PyNULL) - b = Ref(C.PyNULL) - C.PyErr_Fetch(t, v, b) - (pynew(t[]), pynew(v[]), pynew(b[])) + @pyregion begin + t = Ref(C.PyNULL) + v = Ref(C.PyNULL) + b = Ref(C.PyNULL) + C.PyErr_Fetch(t, v, b) + (pynew(t[]), pynew(v[]), pynew(b[])) + end end -errset(t::Py) = C.PyErr_SetNone(t) -errset(t::Py, v::Py) = C.PyErr_SetObject(t, v) -errset(t::Py, v::String) = C.PyErr_SetString(t, v) +errset(t::Py) = @pyregion C.PyErr_SetNone(t) +errset(t::Py, v::Py) = @pyregion C.PyErr_SetObject(t, v) +errset(t::Py, v::String) = @pyregion C.PyErr_SetString(t, v) function errnormalize!(t::Py, v::Py, b::Py) - tref = Ref(getptr(t)) - vref = Ref(getptr(v)) - bref = Ref(getptr(b)) - C.PyErr_NormalizeException(tref, vref, bref) - setptr!(t, tref[]) - setptr!(v, vref[]) - setptr!(b, bref[]) - (t, v, b) + @pyregion begin + tref = Ref(getptr(t)) + vref = Ref(getptr(v)) + bref = Ref(getptr(b)) + C.PyErr_NormalizeException(tref, vref, bref) + setptr!(t, tref[]) + setptr!(v, vref[]) + setptr!(b, bref[]) + (t, v, b) + end end function PyException(v::Py = pybuiltins.None) diff --git a/src/GC/GC.jl b/src/GC/GC.jl index 67c43565..5655bf6a 100644 --- a/src/GC/GC.jl +++ b/src/GC/GC.jl @@ -8,6 +8,8 @@ See [`gc`](@ref). module GC using ..C: C +using ..Region +import ..PythonCall: @pyregion if Base.VERSION ≥ v"1.11" eval( @@ -64,15 +66,12 @@ end Free any Python objects waiting to be freed. -These are objects that were finalized from a thread that was not holding the Python -GIL at the time. - -Like most PythonCall functions, this must only be called from the main thread (i.e. the -thread currently holding the Python GIL.) +These are objects finalized while no Python thread state was attached. This explicit +operation safely establishes the state it needs and can be called from any Julia thread. """ function gc() if C.CTX.is_initialized - unsafe_free_queue() + @pyregion unsafe_free_queue() end nothing end @@ -94,8 +93,8 @@ function enqueue(ptr::C.PyPtr) # If C.CTX.is_initialized is false then the Python interpreter hasn't started yet # or has been finalized; either way attempting to free will cause an error. if ptr != C.PyNULL && C.CTX.is_initialized - if C.PyGILState_Check() == 1 - # If the current thread holds the GIL, then we can immediately free. + if Region.has_tstate() + # An attached state lets us immediately free without blocking a finalizer. C.Py_DecRef(ptr) # We may as well also free any other enqueued objects. if !isempty(QUEUE.items) @@ -103,7 +102,7 @@ function enqueue(ptr::C.PyPtr) end else # Otherwise we push the pointer onto the queue to be freed later, either: - # (a) If a future Python object is finalized on the thread holding the GIL + # (a) If a future Python object is finalized with a state already attached # in the branch above. # (b) If the GCHook() object below is finalized in an ordinary GC. # (c) If the user calls PythonCall.GC.gc(). @@ -115,7 +114,7 @@ end function enqueue_all(ptrs) if any(!=(C.PyNULL), ptrs) && C.CTX.is_initialized - if C.PyGILState_Check() == 1 + if Region.has_tstate() for ptr in ptrs if ptr != C.PyNULL C.Py_DecRef(ptr) @@ -150,7 +149,7 @@ end function _gchook_finalizer(x) if C.CTX.is_initialized finalizer(_gchook_finalizer, x) - if !isempty(QUEUE.items) && C.PyGILState_Check() == 1 + if !isempty(QUEUE.items) && Region.has_tstate() unsafe_free_queue() end end diff --git a/src/GIL/GIL.jl b/src/GIL/GIL.jl index f4b386ce..115e1135 100644 --- a/src/GIL/GIL.jl +++ b/src/GIL/GIL.jl @@ -1,7 +1,7 @@ """ module PythonCall.GIL -Handling the Python Global Interpreter Lock. +Compatibility API for Python interaction regions. See [`lock`](@ref), [`@lock`](@ref), [`unlock`](@ref) and [`@unlock`](@ref). @@ -11,7 +11,7 @@ See [`lock`](@ref), [`@lock`](@ref), [`unlock`](@ref) and [`@unlock`](@ref). """ module GIL -using ..C: C +using ..Region if Base.VERSION ≥ v"1.11" eval( @@ -29,11 +29,10 @@ end """ lock(f) -Lock the GIL, compute `f()`, unlock the GIL, then return the result of `f()`. +Compute `f()` in a Python-heavy region and return its result. -Use this to run Python code from threads that do not currently hold the GIL, such as new -threads. Since the main Julia thread holds the GIL by default, you will need to -[`unlock`](@ref) the GIL before using this function. +PythonCall APIs already establish the required Python thread state automatically. This +compatibility function can amortize transitions across several operations. See [`@lock`](@ref) for the macro form. @@ -42,22 +41,18 @@ See [`@lock`](@ref) for the macro form. This function is experimental. Its semantics may be changed without notice. """ function lock(f) - state = C.PyGILState_Ensure() + token = Region.enter_region() try f() finally - C.PyGILState_Release(state) + Region.exit_region(token) end end """ @lock expr -Lock the GIL, compute `expr`, unlock the GIL, then return the result of `expr`. - -Use this to run Python code from threads that do not currently hold the GIL, such as new -threads. Since the main Julia thread holds the GIL by default, you will need to -[`@unlock`](@ref) the GIL before using this function. +Compute `expr` in a Python-heavy region and return its result. The macro equivalent of [`lock`](@ref). @@ -67,11 +62,11 @@ The macro equivalent of [`lock`](@ref). """ macro lock(expr) quote - state = C.PyGILState_Ensure() + token = $Region.enter_region() try $(esc(expr)) finally - C.PyGILState_Release(state) + $Region.exit_region(token) end end end @@ -79,11 +74,8 @@ end """ unlock(f) -Unlock the GIL, compute `f()`, re-lock the GIL, then return the result of `f()`. - -Use this to run non-Python code with the GIL unlocked, so allowing another thread to run -Python code. That other thread can be a Julia thread, which must lock the GIL using -[`lock`](@ref). +Temporarily relinquish Python-related resources, compute `f()`, restore them, and return +the result. Prefer [`PythonCall.@pyregionbreak`](@ref) in new code. See [`@unlock`](@ref) for the macro form. @@ -92,22 +84,19 @@ See [`@unlock`](@ref) for the macro form. This function is experimental. Its semantics may be changed without notice. """ function unlock(f) - state = C.PyEval_SaveThread() + token = Region.enter_break() try f() finally - C.PyEval_RestoreThread(state) + Region.exit_break(token) end end """ @unlock expr -Unlock the GIL, compute `expr`, re-lock the GIL, then return the result of `expr`. - -Use this to run non-Python code with the GIL unlocked, so allowing another thread to run -Python code. That other thread can be a Julia thread, which must lock the GIL using -[`@lock`](@ref). +Temporarily relinquish Python-related resources, compute `expr`, restore them, and return +the result. Prefer [`PythonCall.@pyregionbreak`](@ref) in new code. The macro equivalent of [`unlock`](@ref). @@ -117,11 +106,11 @@ The macro equivalent of [`unlock`](@ref). """ macro unlock(expr) quote - state = C.PyEval_SaveThread() + token = $Region.enter_break() try $(esc(expr)) finally - C.PyEval_RestoreThread(state) + $Region.exit_break(token) end end end diff --git a/src/JlWrap/any.jl b/src/JlWrap/any.jl index a0df83e2..e58c2aa6 100644 --- a/src/JlWrap/any.jl +++ b/src/JlWrap/any.jl @@ -94,12 +94,12 @@ function pyjlany_call_nogil(self, args_::Py, kwargs_::Py) if pylen(kwargs_) > 0 args = pyconvert(Vector{Any}, args_) kwargs = pyconvert(Dict{Symbol,Any}, kwargs_) - ans = pyjl(GIL.@unlock self(args...; kwargs...)) + ans = pyjl(self(args...; kwargs...)) elseif pylen(args_) > 0 args = pyconvert(Vector{Any}, args_) - ans = pyjl(GIL.@unlock self(args...)) + ans = pyjl(self(args...)) else - ans = pyjl(GIL.@unlock self()) + ans = pyjl(self()) end unsafe_pydel(args_) unsafe_pydel(kwargs_) @@ -608,11 +608,8 @@ class Jl(JlBase2): def jl_callback(self, *args, **kwargs): return self._jl_callmethod($(pyjl_methodnum(pyjlany_callback)), args, kwargs) def jl_call_nogil(self, *args, **kwargs): - '''Call this with the given arguments but with the GIL disabled. - - WARNING: This function must not interact with Python at all without re-acquiring - the GIL. - ''' + '''Compatibility alias for calling this Julia object. Python resources are + relinquished automatically while Julia code runs.''' return self._jl_callmethod($(pyjl_methodnum(pyjlany_call_nogil)), args, kwargs) def _repr_mimebundle_(self, include=None, exclude=None): return self._jl_callmethod($(pyjl_methodnum(pyjlany_mimebundle)), include, exclude) diff --git a/src/JlWrap/base.jl b/src/JlWrap/base.jl index 2497f4b0..67723e59 100644 --- a/src/JlWrap/base.jl +++ b/src/JlWrap/base.jl @@ -39,25 +39,25 @@ function Cjl._pyjl_callmethod(f, self_::C.PyPtr, args_::C.PyPtr, nargs::C.Py_ssi try if nargs == 1 in_f = true - ans = f(self)::Py + ans = @pyregionbreak(f(self))::Py in_f = false elseif nargs == 2 arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) in_f = true - ans = f(self, arg1)::Py + ans = @pyregionbreak(f(self, arg1))::Py in_f = false elseif nargs == 3 arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) arg2 = pynew(incref(C.PyTuple_GetItem(args_, 2))) in_f = true - ans = f(self, arg1, arg2)::Py + ans = @pyregionbreak(f(self, arg1, arg2))::Py in_f = false elseif nargs == 4 arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) arg2 = pynew(incref(C.PyTuple_GetItem(args_, 2))) arg3 = pynew(incref(C.PyTuple_GetItem(args_, 3))) in_f = true - ans = f(self, arg1, arg2, arg3)::Py + ans = @pyregionbreak(f(self, arg1, arg2, arg3))::Py in_f = false else errset( diff --git a/src/PyMacro/PyMacro.jl b/src/PyMacro/PyMacro.jl index d8dc45ab..a185ea8e 100644 --- a/src/PyMacro/PyMacro.jl +++ b/src/PyMacro/PyMacro.jl @@ -560,7 +560,7 @@ function py_macro_lower(st, body, ans, ex; flavour = :expr) # @jl x elseif @capture(ex, @jl ax_) y = py_macro_lower_jl(st, ax) - py_macro_assign(body, ans, y) + py_macro_assign(body, ans, :(@pyregionbreak $y)) return false # @compile code mode=mode ... @@ -918,7 +918,21 @@ Py({'y': 2}) This macro is experimental. It may be modified or removed in a future release. """ macro py(ex) - esc(py_macro(ex, __module__, __source__)) + lowered = py_macro(ex, __module__, __source__) + assigned = Symbol[] + function find_assigned(x) + x isa Expr || return + if x.head === :(=) && x.args[1] isa Symbol + push!(assigned, x.args[1]) + end + foreach(find_assigned, x.args) + end + find_assigned(lowered) + locals = unique(assigned) + esc(quote + $(isempty(locals) ? nothing : Expr(:local, locals...)) + @pyregion $lowered + end) end end diff --git a/src/PythonCall.jl b/src/PythonCall.jl index c6b166bd..48da84ba 100644 --- a/src/PythonCall.jl +++ b/src/PythonCall.jl @@ -6,6 +6,7 @@ include("API/API.jl") include("Utils/Utils.jl") include("NumpyDates/NumpyDates.jl") include("C/C.jl") +include("Region/Region.jl") include("GIL/GIL.jl") include("GC/GC.jl") include("Core/Core.jl") @@ -15,6 +16,10 @@ include("Wrap/Wrap.jl") include("JlWrap/JlWrap.jl") include("Compat/Compat.jl") +function __init__() + Region.start_runtime() +end + # not API but used in tests for k in [ :pyjlanytype, diff --git a/src/Region/Region.jl b/src/Region/Region.jl new file mode 100644 index 00000000..72ff5871 --- /dev/null +++ b/src/Region/Region.jl @@ -0,0 +1,253 @@ +"""Internal task-safe management of CPython thread states.""" +module Region + +using ..C: C +import ..PythonCall: @pyregion, @pyregionbreak + +const PyThreadStatePtr = Ptr{Cvoid} + +mutable struct ThreadState + sem::Base.Semaphore + tstate::PyThreadStatePtr +end +ThreadState() = ThreadState(Base.Semaphore(1), C_NULL) + +mutable struct TaskState + tstate::PyThreadStatePtr + sem::Union{Nothing,Base.Semaphore} + attached::Bool + tid::Int + oldsticky::Bool +end +TaskState() = TaskState(C_NULL, nothing, false, 0, false) + +const THREAD_STATE = Base.OncePerThread(ThreadState) +const TASK_STATE = Base.OncePerTask(TaskState) +const INTERP = Ref{Ptr{Cvoid}}(C_NULL) + +current_tstate() = C.PyThreadState_GetUnchecked() +has_tstate() = C.CTX.is_initialized && current_tstate() != C_NULL + +function reset!(s::TaskState) + s.tstate = C_NULL + s.sem = nothing + s.attached = false + s.tid = 0 + return +end + +function start_session!(s::TaskState) + task = current_task() + s.oldsticky = task.sticky + task.sticky = true + s.tid = Threads.threadid() + ts = THREAD_STATE() + s.sem = ts.sem + return ts +end + +check_thread(s::TaskState) = (@assert current_task().sticky && Threads.threadid() == s.tid) + +function enter_region() + s = TASK_STATE() + if s.tstate != C_NULL + check_thread(s) + if s.attached + return :noop + end + Base.acquire(s.sem::Base.Semaphore) + try + C.PyEval_RestoreThread(s.tstate) + s.attached = true + catch + Base.release(s.sem::Base.Semaphore) + rethrow() + end + return :detach + end + + task = current_task() + ts = try + start_session!(s) + catch + task.sticky = s.oldsticky + reset!(s) + rethrow() + end + acquired = false + try + Base.acquire(ts.sem) + acquired = true + current = current_tstate() + if current != C_NULL + s.tstate = current + s.attached = true + return :root_borrowed + end + if ts.tstate == C_NULL + ts.tstate = C.PyThreadState_New(INTERP[]) + ts.tstate == C_NULL && error("PyThreadState_New failed") + end + s.tstate = ts.tstate + C.PyEval_RestoreThread(s.tstate) + s.attached = true + return :root_owned + catch + acquired && Base.release(ts.sem) + task.sticky = s.oldsticky + reset!(s) + rethrow() + end +end + +function exit_region(token) + token === :noop && return + s = TASK_STATE() + check_thread(s) + if token === :detach + saved = C.PyEval_SaveThread() + @assert saved == s.tstate + s.attached = false + Base.release(s.sem::Base.Semaphore) + elseif token === :root_owned + saved = C.PyEval_SaveThread() + @assert saved == s.tstate + sem, oldsticky = s.sem::Base.Semaphore, s.oldsticky + reset!(s) + Base.release(sem) + current_task().sticky = oldsticky + elseif token === :root_borrowed + @assert current_tstate() == s.tstate + sem, oldsticky = s.sem::Base.Semaphore, s.oldsticky + reset!(s) + Base.release(sem) + current_task().sticky = oldsticky + else + error("invalid Python region token") + end + return +end + +function enter_break() + s = TASK_STATE() + if s.tstate != C_NULL + check_thread(s) + !s.attached && return :noop + saved = C.PyEval_SaveThread() + @assert saved == s.tstate + s.attached = false + Base.release(s.sem::Base.Semaphore) + return :restore + end + current_tstate() == C_NULL && return :noop + + task = current_task() + ts = try + start_session!(s) + catch + task.sticky = s.oldsticky + reset!(s) + rethrow() + end + acquired = false + try + Base.acquire(ts.sem) + acquired = true + current = current_tstate() + if current == C_NULL + Base.release(ts.sem) + task.sticky = s.oldsticky + reset!(s) + return :noop + end + s.tstate = current + saved = C.PyEval_SaveThread() + @assert saved == current + s.attached = false + Base.release(ts.sem) + return :root_restore + catch + acquired && Base.release(ts.sem) + task.sticky = s.oldsticky + reset!(s) + rethrow() + end +end + +function exit_break(token) + token === :noop && return + s = TASK_STATE() + check_thread(s) + Base.acquire(s.sem::Base.Semaphore) + C.PyEval_RestoreThread(s.tstate) + s.attached = true + if token === :root_restore + sem, oldsticky = s.sem::Base.Semaphore, s.oldsticky + reset!(s) + Base.release(sem) + current_task().sticky = oldsticky + elseif token !== :restore + error("invalid Python region-break token") + end + return +end + +""" + @pyregion expr + +Mark `expr` as relatively straight-line, Python-heavy work. PythonCall APIs manage the +Python resources they need automatically; this optional region can amortize that work across +several operations. Regions nest freely. Put Julia code which deliberately yields, waits, or +blocks cooperatively in [`@pyregionbreak`](@ref). +""" +macro pyregion(ex) + quote + local token = $enter_region() + try + $(esc(ex)) + finally + $exit_region(token) + end + end +end + +""" + @pyregionbreak expr + +Mark `expr` as a section where Python interaction is absent or infrequent, allowing an +enclosing [`@pyregion`](@ref) to relinquish Python-related resources temporarily. Nested +PythonCall operations still work automatically, and both kinds of region nest freely. +""" +macro pyregionbreak(ex) + quote + local token = $enter_break() + try + $(esc(ex)) + finally + $exit_break(token) + end + end +end + +function __init__() + current = current_tstate() + current == C_NULL && error("Python initialization did not leave an attached thread state") + INTERP[] = C.PyThreadState_GetInterpreter(current) + INTERP[] == C_NULL && error("could not determine the Python interpreter") + return +end + +function start_runtime() + if !C.CTX.is_embedded && !C.CTX.is_preinitialized + C.PyEval_SaveThread() + C.FINALIZE_HOOK[] = function () + ts = THREAD_STATE() + ts.tstate == C_NULL && (ts.tstate = C.PyThreadState_New(INTERP[])) + C.PyEval_RestoreThread(ts.tstate) + C.Py_FinalizeEx() == -1 && @warn "Py_FinalizeEx() error" + end + end + return +end + +end diff --git a/test/GC.jl b/test/GC.jl index f180cfaf..f305c6f7 100644 --- a/test/GC.jl +++ b/test/GC.jl @@ -29,5 +29,9 @@ end VERSION >= v"1.10.0-" && @test !isempty(PythonCall.GC.QUEUE.items) GC.gc() + # A Julia finalizer must never attach a Python thread state merely to decref. + @test PythonCall.C.PyThreadState_GetUnchecked() == C_NULL + Threads.nthreads() > 1 && @test !isempty(PythonCall.GC.QUEUE.items) + PythonCall.GC.gc() @test isempty(PythonCall.GC.QUEUE.items) end diff --git a/test/Region.jl b/test/Region.jl new file mode 100644 index 00000000..b000ad52 --- /dev/null +++ b/test/Region.jl @@ -0,0 +1,42 @@ +using TestItemRunner + +@testitem "Python regions and task-safe thread states" setup = [Setup] begin + using Base.Threads + + @test PythonCall.C.PyThreadState_GetUnchecked() == C_NULL + @test @pyregion pyconvert(Int, pyint(12)) == 12 + @test PythonCall.C.PyThreadState_GetUnchecked() == C_NULL + + @test @pyregion begin + @pyregion pyconvert(Int, pyint(1)) == 1 + @pyregionbreak begin + yield() + @pyregion pyconvert(Int, pyint(2)) == 2 + @pyregionbreak yield() + end + pyconvert(Int, pyint(3)) == 3 + end + + @test_throws ErrorException @pyregion @pyregionbreak error("region exception") + @test PythonCall.C.PyThreadState_GetUnchecked() == C_NULL + @test_throws PyException @pyregion pybuiltins.int("not an integer") + @test pyconvert(Int, pyint(5)) == 5 + + results = fetch.([@spawn begin + total = 0 + for i in 1:100 + total += pyconvert(Int, pyint(i)) + @pyregionbreak yield() + end + total + end for _ in 1:max(8, 2nthreads())]) + @test all(==(5050), results) + + f = pyfunc() do + yield() + pyconvert(Int, pybuiltins.sum([1, 2, 3])) + end + @test pyconvert(Int, f()) == 6 + + @test pyconvert(Int, @py 1 + @jl(pyconvert(Int, pyint(2)))) == 3 +end From d35fa84d4e01767b2f981898348775133fec9abe Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Sat, 19 Sep 2026 22:08:30 +0100 Subject: [PATCH 2/7] Restore Julia 1.10 thread state compatibility --- AGENTS.md | 3 --- Project.toml | 2 +- pysrc/juliacall/juliapkg-dev.json | 2 +- pysrc/juliacall/juliapkg.json | 2 +- pytest/test_all.py | 2 ++ src/Region/Region.jl | 5 +++-- src/Utils/Utils.jl | 36 +++++++++++++++++++++++++++++++ test/Utils.jl | 23 ++++++++++++++++++++ 8 files changed, 67 insertions(+), 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 236a5279..ce85f833 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,9 +6,6 @@ - Copy `pysrc/juliacall/juliapkg-dev.json` to `pysrc/juliacall/juliapkg.json` before running (do **not** commit this copy). - Execute with `uv run pytest -s --nbval ./pytest` (add `--cov=pysrc` when coverage is needed). - Sometimes `juliapkg` requires Julia 1.10–1.11; `juliaup` already provides 1.11.7 in this environment. -- With Python 3.14, juliapkg's OpenSSL compatibility currently constrains Julia to 1.11 or - older. Packages requiring Julia 1.12+ therefore cannot run the Python suite under that - Python; use an older Python whose OpenSSL constraint permits Julia 1.12+. The majority of tests live in the Julia package; Python tests cover functionality that cannot be exercised from Julia (e.g., JuliaCall-specific behavior). Run both suites—typically Julia first—in whichever order makes sense. diff --git a/Project.toml b/Project.toml index 970b2162..9e6d15fe 100644 --- a/Project.toml +++ b/Project.toml @@ -34,7 +34,7 @@ PyCall = "1" Serialization = "1" Tables = "1" UnsafePointers = "1" -julia = "1.12" +julia = "1.10" [extras] CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" diff --git a/pysrc/juliacall/juliapkg-dev.json b/pysrc/juliacall/juliapkg-dev.json index 3dd2bbd1..3e6910a8 100644 --- a/pysrc/juliacall/juliapkg-dev.json +++ b/pysrc/juliacall/juliapkg-dev.json @@ -1,5 +1,5 @@ { - "julia": "^1.12", + "julia": "^1.10.3", "packages": { "PythonCall": { "uuid": "6099a3de-0909-46bc-b1f4-468b9a2dfc0d", diff --git a/pysrc/juliacall/juliapkg.json b/pysrc/juliacall/juliapkg.json index 37f68861..f45ce3b1 100644 --- a/pysrc/juliacall/juliapkg.json +++ b/pysrc/juliacall/juliapkg.json @@ -1,5 +1,5 @@ { - "julia": "^1.12", + "julia": "^1.10.3", "packages": { "PythonCall": { "uuid": "6099a3de-0909-46bc-b1f4-468b9a2dfc0d", diff --git a/pytest/test_all.py b/pytest/test_all.py index 7ed9fbb5..0267f7df 100644 --- a/pytest/test_all.py +++ b/pytest/test_all.py @@ -105,6 +105,8 @@ def test_julia_gc(): end end GC.gc() + @test !isempty(PythonCall.GC.QUEUE.items) + PythonCall.GC.gc() @test isempty(PythonCall.GC.QUEUE.items) """ ) diff --git a/src/Region/Region.jl b/src/Region/Region.jl index 72ff5871..d69e1fab 100644 --- a/src/Region/Region.jl +++ b/src/Region/Region.jl @@ -2,6 +2,7 @@ module Region using ..C: C +using ..Utils import ..PythonCall: @pyregion, @pyregionbreak const PyThreadStatePtr = Ptr{Cvoid} @@ -21,8 +22,8 @@ mutable struct TaskState end TaskState() = TaskState(C_NULL, nothing, false, 0, false) -const THREAD_STATE = Base.OncePerThread(ThreadState) -const TASK_STATE = Base.OncePerTask(TaskState) +const THREAD_STATE = Utils.OncePerThread{ThreadState}(ThreadState) +const TASK_STATE = Utils.OncePerTask{TaskState}(TaskState) const INTERP = Ref{Ptr{Cvoid}}(C_NULL) current_tstate() = C.PyThreadState_GetUnchecked() diff --git a/src/Utils/Utils.jl b/src/Utils/Utils.jl index ea10a115..be5d2a5a 100644 --- a/src/Utils/Utils.jl +++ b/src/Utils/Utils.jl @@ -2,6 +2,42 @@ module Utils using Preferences: @load_preference +# Minimal package-local equivalents of Base.OncePerThread and Base.OncePerTask. The Base +# implementations are only available on newer Julia releases, while PythonCall supports +# Julia 1.10. These intentionally provide only the callable interface needed internally. +mutable struct OncePerThread{T,F} + initializer::F + values::Dict{Int,T} + lock::ReentrantLock +end + +OncePerThread{T}(initializer::F) where {T,F} = + OncePerThread{T,F}(initializer, Dict{Int,T}(), ReentrantLock()) + +function (once::OncePerThread{T})() where {T} + tid = Threads.threadid() + lock(once.lock) + try + return get!(once.values, tid) do + once.initializer()::T + end + finally + unlock(once.lock) + end +end + +mutable struct OncePerTask{T,F} + initializer::F +end + +OncePerTask{T}(initializer::F) where {T,F} = OncePerTask{T,F}(initializer) + +function (once::OncePerTask{T})() where {T} + get!(task_local_storage(), once) do + once.initializer()::T + end::T +end + function getpref(::Type{T}, prefname, envname, default = nothing) where {T} ans = @load_preference(prefname, nothing) ans === nothing || return checkpref(T, ans)::T diff --git a/test/Utils.jl b/test/Utils.jl index 33c82bd9..3b49a28d 100644 --- a/test/Utils.jl +++ b/test/Utils.jl @@ -22,3 +22,26 @@ end @test s[1:2] == "ab" @test s[1:2:end] == "aaaab" end + +@testitem "OncePerThread and OncePerTask" begin + thread_count = Threads.Atomic{Int}(0) + per_thread = PythonCall.Utils.OncePerThread{Int}() do + Threads.atomic_add!(thread_count, 1) + Threads.threadid() + end + @test per_thread() == Threads.threadid() + @test per_thread() == Threads.threadid() + @test thread_count[] == 1 + + task_count = Threads.Atomic{Int}(0) + per_task = PythonCall.Utils.OncePerTask{UInt}() do + Threads.atomic_add!(task_count, 1) + objectid(current_task()) + end + @test per_task() == per_task() + other_per_task = PythonCall.Utils.OncePerTask{UInt}(() -> typemax(UInt)) + @test other_per_task() == typemax(UInt) + other = fetch(Threads.@spawn (per_task(), per_task())) + @test other[1] == other[2] + @test task_count[] == 2 +end From 8f82c46fb6672dae947da1aa2aec85b8f94f63d1 Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Mon, 21 Sep 2026 09:18:38 +0100 Subject: [PATCH 3/7] Keep Python error handling within regions --- src/Convert/pyconvert.jl | 49 ++++--- src/Core/builtins.jl | 274 +++++++++++++++++++++------------------ src/Core/err.jl | 60 +++++---- src/JlWrap/any.jl | 10 +- src/JlWrap/array.jl | 40 +++--- src/JlWrap/base.jl | 106 +++++++-------- src/JlWrap/io.jl | 10 +- src/JlWrap/set.jl | 6 +- src/JlWrap/vector.jl | 12 +- 9 files changed, 299 insertions(+), 268 deletions(-) diff --git a/src/Convert/pyconvert.jl b/src/Convert/pyconvert.jl index 10435f11..ab69aad9 100644 --- a/src/Convert/pyconvert.jl +++ b/src/Convert/pyconvert.jl @@ -369,13 +369,15 @@ On failure, evaluates to `onfail`, which defaults to `return pyconvert_unconvert """ macro pyconvert(T, x, onfail = :(return $pyconvert_unconverted())) quote - T = $(esc(T)) - x = $(esc(x)) - ans = pytryconvert(T, x) - if pyconvert_isunconverted(ans) - $(esc(onfail)) - else - pyconvert_result(T, ans) + @pyregion begin + T = $(esc(T)) + x = $(esc(x)) + ans = pytryconvert(T, x) + if pyconvert_isunconverted(ans) + $(esc(onfail)) + else + pyconvert_result(T, ans) + end end end end @@ -387,10 +389,19 @@ Convert the Python object `x` to a `T`. If `d` is specified, it is returned on failure instead of throwing an error. """ -pyconvert(::Type{T}, x) where {T} = @autopy x @pyconvert T x_ error( - "cannot convert this Python '$(pytype(x_).__name__)' to a Julia '$T'", -) -pyconvert(::Type{T}, x, d) where {T} = @autopy x @pyconvert T x_ d +function pyconvert(::Type{T}, x) where {T} + @pyregion begin + @autopy x @pyconvert T x_ error( + "cannot convert this Python '$(pytype(x_).__name__)' to a Julia '$T'", + ) + end +end + +function pyconvert(::Type{T}, x, d) where {T} + @pyregion begin + @autopy x @pyconvert T x_ d + end +end """ pyconvertarg(T, x, name) @@ -399,12 +410,16 @@ Convert the Python object `x` to a `T`. On failure, throws a Python `TypeError` saying that the argument `name` could not be converted. """ -pyconvertarg(::Type{T}, x, name) where {T} = @autopy x @pyconvert T x_ begin - errset( - pybuiltins.TypeError, - "Cannot convert argument '$name' to a Julia '$T', got a '$(pytype(x_).__name__)'", - ) - pythrow() +function pyconvertarg(::Type{T}, x, name) where {T} + @pyregion begin + @autopy x @pyconvert T x_ begin + errset( + pybuiltins.TypeError, + "Cannot convert argument '$name' to a Julia '$T', got a '$(pytype(x_).__name__)'", + ) + pythrow() + end + end end function init_pyconvert() diff --git a/src/Core/builtins.jl b/src/Core/builtins.jl index 13faca50..29a07e43 100644 --- a/src/Core/builtins.jl +++ b/src/Core/builtins.jl @@ -5,7 +5,7 @@ True if `x` and `y` are the same Python object. Equivalent to `x is y` in Python. """ -pyis(x, y) = @autopy x y getptr(x_) == getptr(y_) +pyis(x, y) = @pyregion @autopy x y getptr(x_) == getptr(y_) pyisnot(x, y) = !pyis(x, y) @@ -14,7 +14,7 @@ pyisnot(x, y) = !pyis(x, y) Equivalent to `repr(x)` in Python. """ -pyrepr(x) = pynew(errcheck(@autopy x C.PyObject_Repr(x_))) +pyrepr(x) = @pyregion pynew(errcheck(@autopy x C.PyObject_Repr(x_))) pyrepr(::Type{String}, x) = (s = pyrepr(x); ans = pystr_asstring(s); unsafe_pydel(s); ans) """ @@ -22,7 +22,7 @@ pyrepr(::Type{String}, x) = (s = pyrepr(x); ans = pystr_asstring(s); unsafe_pyde Equivalent to `ascii(x)` in Python. """ -pyascii(x) = pynew(errcheck(@autopy x C.PyObject_ASCII(x_))) +pyascii(x) = @pyregion pynew(errcheck(@autopy x C.PyObject_ASCII(x_))) pyascii(::Type{String}, x) = (s = pyascii(x); ans = pystr_asstring(s); unsafe_pydel(s); ans) """ @@ -33,17 +33,19 @@ Equivalent to `hasattr(x, k)` in Python. Tests if `getattr(x, k)` raises an `AttributeError`. """ function pyhasattr(x, k) - ptr = @autopy x k C.PyObject_GetAttr(x_, k_) - if iserrset(ptr) - if errmatches(pybuiltins.AttributeError) - errclear() - return false + @pyregion begin + ptr = @autopy x k C.PyObject_GetAttr(x_, k_) + if iserrset(ptr) + if errmatches(pybuiltins.AttributeError) + errclear() + return false + else + pythrow() + end else - pythrow() + decref(ptr) + return true end - else - decref(ptr) - return true end end # pyhasattr(x, k) = errcheck(@autopy x k C.PyObject_HasAttr(x_, k_)) == 1 @@ -55,18 +57,20 @@ Equivalent to `getattr(x, k)` or `x.k` in Python. If `d` is specified, it is returned if the attribute does not exist. """ -pygetattr(x, k) = pynew(errcheck(@autopy x k C.PyObject_GetAttr(x_, k_))) +pygetattr(x, k) = @pyregion pynew(errcheck(@autopy x k C.PyObject_GetAttr(x_, k_))) function pygetattr(x, k, d) - ptr = @autopy x k C.PyObject_GetAttr(x_, k_) - if iserrset(ptr) - if errmatches(pybuiltins.AttributeError) - errclear() - return d + @pyregion begin + ptr = @autopy x k C.PyObject_GetAttr(x_, k_) + if iserrset(ptr) + if errmatches(pybuiltins.AttributeError) + errclear() + return d + else + pythrow() + end else - pythrow() + return pynew(ptr) end - else - return pynew(ptr) end end @@ -75,56 +79,56 @@ end Equivalent to `setattr(x, k, v)` or `x.k = v` in Python. """ -pysetattr(x, k, v) = (errcheck(@autopy x k v C.PyObject_SetAttr(x_, k_, v_)); nothing) +pysetattr(x, k, v) = @pyregion (errcheck(@autopy x k v C.PyObject_SetAttr(x_, k_, v_)); nothing) """ pydelattr(x, k) Equivalent to `delattr(x, k)` or `del x.k` in Python. """ -pydelattr(x, k) = (errcheck(@autopy x k C.PyObject_SetAttr(x_, k_, C.PyNULL)); nothing) +pydelattr(x, k) = @pyregion (errcheck(@autopy x k C.PyObject_SetAttr(x_, k_, C.PyNULL)); nothing) """ pyissubclass(s, t) Test if `s` is a subclass of `t`. Equivalent to `issubclass(s, t)` in Python. """ -pyissubclass(s, t) = errcheck(@autopy s t C.PyObject_IsSubclass(s_, t_)) == 1 +pyissubclass(s, t) = @pyregion errcheck(@autopy s t C.PyObject_IsSubclass(s_, t_)) == 1 """ pyisinstance(x, t) Test if `x` is of type `t`. Equivalent to `isinstance(x, t)` in Python. """ -pyisinstance(x, t) = errcheck(@autopy x t C.PyObject_IsInstance(x_, t_)) == 1 +pyisinstance(x, t) = @pyregion errcheck(@autopy x t C.PyObject_IsInstance(x_, t_)) == 1 """ pyhash(x) Equivalent to `hash(x)` in Python, converted to an `Integer`. """ -pyhash(x) = errcheck(@autopy x C.PyObject_Hash(x_)) +pyhash(x) = @pyregion errcheck(@autopy x C.PyObject_Hash(x_)) """ pytruth(x) The truthyness of `x`. Equivalent to `bool(x)` in Python, converted to a `Bool`. """ -pytruth(x) = errcheck(@autopy x C.PyObject_IsTrue(x_)) == 1 +pytruth(x) = @pyregion errcheck(@autopy x C.PyObject_IsTrue(x_)) == 1 """ pynot(x) The falsyness of `x`. Equivalent to `not x` in Python, converted to a `Bool`. """ -pynot(x) = errcheck(@autopy x C.PyObject_Not(x_)) == 1 +pynot(x) = @pyregion errcheck(@autopy x C.PyObject_Not(x_)) == 1 """ pylen(x) The length of `x`. Equivalent to `len(x)` in Python, converted to an `Integer`. """ -pylen(x) = errcheck(@autopy x C.PyObject_Length(x_)) +pylen(x) = @pyregion errcheck(@autopy x C.PyObject_Length(x_)) """ pyhasitem(x, k) @@ -132,17 +136,19 @@ pylen(x) = errcheck(@autopy x C.PyObject_Length(x_)) Test if `pygetitem(x, k)` raises a `KeyError` or `AttributeError`. """ function pyhasitem(x, k) - ptr = @autopy x k C.PyObject_GetItem(x_, k_) - if iserrset(ptr) - if errmatches(pybuiltins.KeyError) || errmatches(pybuiltins.IndexError) - errclear() - return false + @pyregion begin + ptr = @autopy x k C.PyObject_GetItem(x_, k_) + if iserrset(ptr) + if errmatches(pybuiltins.KeyError) || errmatches(pybuiltins.IndexError) + errclear() + return false + else + pythrow() + end else - pythrow() + decref(ptr) + return true end - else - decref(ptr) - return true end end @@ -154,18 +160,20 @@ Equivalent `x[k]` in Python. If `d` is specified, it is returned if the item does not exist (i.e. if `x[k]` raises a `KeyError` or `IndexError`). """ -pygetitem(x, k) = pynew(errcheck(@autopy x k C.PyObject_GetItem(x_, k_))) +pygetitem(x, k) = @pyregion pynew(errcheck(@autopy x k C.PyObject_GetItem(x_, k_))) function pygetitem(x, k, d) - ptr = @autopy x k C.PyObject_GetItem(x_, k_) - if iserrset(ptr) - if errmatches(pybuiltins.KeyError) || errmatches(pybuiltins.IndexError) - errclear() - return d + @pyregion begin + ptr = @autopy x k C.PyObject_GetItem(x_, k_) + if iserrset(ptr) + if errmatches(pybuiltins.KeyError) || errmatches(pybuiltins.IndexError) + errclear() + return d + else + pythrow() + end else - pythrow() + return pynew(ptr) end - else - return pynew(ptr) end end @@ -174,26 +182,26 @@ end Equivalent to `setitem(x, k, v)` or `x[k] = v` in Python. """ -pysetitem(x, k, v) = (errcheck(@autopy x k v C.PyObject_SetItem(x_, k_, v_)); nothing) +pysetitem(x, k, v) = @pyregion (errcheck(@autopy x k v C.PyObject_SetItem(x_, k_, v_)); nothing) """ pydelitem(x, k) Equivalent to `delitem(x, k)` or `del x[k]` in Python. """ -pydelitem(x, k) = (errcheck(@autopy x k C.PyObject_DelItem(x_, k_)); nothing) +pydelitem(x, k) = @pyregion (errcheck(@autopy x k C.PyObject_DelItem(x_, k_)); nothing) """ pydir(x) Equivalent to `dir(x)` in Python. """ -pydir(x) = pynew(errcheck(@autopy x C.PyObject_Dir(x_))) +pydir(x) = @pyregion pynew(errcheck(@autopy x C.PyObject_Dir(x_))) -pycallargs(f) = pynew(errcheck(@autopy f C.PyObject_CallObject(f_, C.PyNULL))) -pycallargs(f, args) = pynew(errcheck(@autopy f args C.PyObject_CallObject(f_, args_))) +pycallargs(f) = @pyregion pynew(errcheck(@autopy f C.PyObject_CallObject(f_, C.PyNULL))) +pycallargs(f, args) = @pyregion pynew(errcheck(@autopy f args C.PyObject_CallObject(f_, args_))) pycallargs(f, args, kwargs) = - pynew(errcheck(@autopy f args kwargs C.PyObject_Call(f_, args_, kwargs_))) + @pyregion pynew(errcheck(@autopy f args kwargs C.PyObject_Call(f_, args_, kwargs_))) """ pycall(f, args...; kwargs...) @@ -223,7 +231,7 @@ pycall(f, args...; kwargs...) = Equivalent to `x == y` in Python. The second form converts to `Bool`. """ -pyeq(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_EQ))) +pyeq(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_EQ))) """ pyne(x, y) @@ -231,7 +239,7 @@ pyeq(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_EQ)) Equivalent to `x != y` in Python. The second form converts to `Bool`. """ -pyne(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_NE))) +pyne(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_NE))) """ pyle(x, y) @@ -239,7 +247,7 @@ pyne(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_NE)) Equivalent to `x <= y` in Python. The second form converts to `Bool`. """ -pyle(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LE))) +pyle(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LE))) """ pylt(x, y) @@ -247,7 +255,7 @@ pyle(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LE)) Equivalent to `x < y` in Python. The second form converts to `Bool`. """ -pylt(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LT))) +pylt(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LT))) """ pyge(x, y) @@ -255,7 +263,7 @@ pylt(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_LT)) Equivalent to `x >= y` in Python. The second form converts to `Bool`. """ -pyge(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_GE))) +pyge(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_GE))) """ pygt(x, y) @@ -263,26 +271,26 @@ pyge(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_GE)) Equivalent to `x > y` in Python. The second form converts to `Bool`. """ -pygt(x, y) = pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_GT))) +pygt(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyObject_RichCompare(x_, y_, C.Py_GT))) pyeq(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_EQ)) == 1 + @pyregion errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_EQ)) == 1 pyne(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_NE)) == 1 + @pyregion errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_NE)) == 1 pyle(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_LE)) == 1 + @pyregion errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_LE)) == 1 pylt(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_LT)) == 1 + @pyregion errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_LT)) == 1 pyge(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_GE)) == 1 + @pyregion errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_GE)) == 1 pygt(::Type{Bool}, x, y) = - errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_GT)) == 1 + @pyregion errcheck(@autopy x y C.PyObject_RichCompareBool(x_, y_, C.Py_GT)) == 1 """ pycontains(x, v) Equivalent to `v in x` in Python. """ -pycontains(x, v) = errcheck(@autopy x v C.PySequence_Contains(x_, v_)) == 1 +pycontains(x, v) = @pyregion errcheck(@autopy x v C.PySequence_Contains(x_, v_)) == 1 """ pyin(v, x) @@ -301,31 +309,31 @@ pynotin(v, x) = !pyin(v, x) Equivalent to `-x` in Python. """ -pyneg(x) = pynew(errcheck(@autopy x C.PyNumber_Negative(x_))) +pyneg(x) = @pyregion pynew(errcheck(@autopy x C.PyNumber_Negative(x_))) """ pypos(x) Equivalent to `+x` in Python. """ -pypos(x) = pynew(errcheck(@autopy x C.PyNumber_Positive(x_))) +pypos(x) = @pyregion pynew(errcheck(@autopy x C.PyNumber_Positive(x_))) """ pyabs(x) Equivalent to `abs(x)` in Python. """ -pyabs(x) = pynew(errcheck(@autopy x C.PyNumber_Absolute(x_))) +pyabs(x) = @pyregion pynew(errcheck(@autopy x C.PyNumber_Absolute(x_))) """ pyinv(x) Equivalent to `~x` in Python. """ -pyinv(x) = pynew(errcheck(@autopy x C.PyNumber_Invert(x_))) +pyinv(x) = @pyregion pynew(errcheck(@autopy x C.PyNumber_Invert(x_))) """ pyindex(x) Convert `x` losslessly to an `int`. """ -pyindex(x) = pynew(errcheck(@autopy x C.PyNumber_Index(x_))) +pyindex(x) = @pyregion pynew(errcheck(@autopy x C.PyNumber_Index(x_))) # binary """ @@ -333,79 +341,79 @@ pyindex(x) = pynew(errcheck(@autopy x C.PyNumber_Index(x_))) Equivalent to `x + y` in Python. """ -pyadd(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Add(x_, y_))) +pyadd(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Add(x_, y_))) """ pysub(x, y) Equivalent to `x - y` in Python. """ -pysub(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Subtract(x_, y_))) +pysub(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Subtract(x_, y_))) """ pymul(x, y) Equivalent to `x * y` in Python. """ -pymul(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Multiply(x_, y_))) +pymul(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Multiply(x_, y_))) """ pymatmul(x, y) Equivalent to `x @ y` in Python. """ -pymatmul(x, y) = pynew(errcheck(@autopy x y C.PyNumber_MatrixMultiply(x_, y_))) +pymatmul(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_MatrixMultiply(x_, y_))) """ pyfloordiv(x, y) Equivalent to `x // y` in Python. """ -pyfloordiv(x, y) = pynew(errcheck(@autopy x y C.PyNumber_FloorDivide(x_, y_))) +pyfloordiv(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_FloorDivide(x_, y_))) """ pytruediv(x, y) Equivalent to `x / y` in Python. """ -pytruediv(x, y) = pynew(errcheck(@autopy x y C.PyNumber_TrueDivide(x_, y_))) +pytruediv(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_TrueDivide(x_, y_))) """ pymod(x, y) Equivalent to `x % y` in Python. """ -pymod(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Remainder(x_, y_))) +pymod(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Remainder(x_, y_))) """ pydivmod(x, y) Equivalent to `divmod(x, y)` in Python. """ -pydivmod(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Divmod(x_, y_))) +pydivmod(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Divmod(x_, y_))) """ pylshift(x, y) Equivalent to `x << y` in Python. """ -pylshift(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Lshift(x_, y_))) +pylshift(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Lshift(x_, y_))) """ pyrshift(x, y) Equivalent to `x >> y` in Python. """ -pyrshift(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Rshift(x_, y_))) +pyrshift(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Rshift(x_, y_))) """ pyand(x, y) Equivalent to `x & y` in Python. """ -pyand(x, y) = pynew(errcheck(@autopy x y C.PyNumber_And(x_, y_))) +pyand(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_And(x_, y_))) """ pyxor(x, y) Equivalent to `x ^ y` in Python. """ -pyxor(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Xor(x_, y_))) +pyxor(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Xor(x_, y_))) """ pyor(x, y) Equivalent to `x | y` in Python. """ -pyor(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Or(x_, y_))) +pyor(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_Or(x_, y_))) # binary in-place """ @@ -413,73 +421,73 @@ pyor(x, y) = pynew(errcheck(@autopy x y C.PyNumber_Or(x_, y_))) In-place add. `x = pyiadd(x, y)` is equivalent to `x += y` in Python. """ -pyiadd(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceAdd(x_, y_))) +pyiadd(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceAdd(x_, y_))) """ pyisub(x, y) In-place subtract. `x = pyisub(x, y)` is equivalent to `x -= y` in Python. """ -pyisub(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceSubtract(x_, y_))) +pyisub(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceSubtract(x_, y_))) """ pyimul(x, y) In-place multiply. `x = pyimul(x, y)` is equivalent to `x *= y` in Python. """ -pyimul(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceMultiply(x_, y_))) +pyimul(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceMultiply(x_, y_))) """ pyimatmul(x, y) In-place matrix multiply. `x = pyimatmul(x, y)` is equivalent to `x @= y` in Python. """ -pyimatmul(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceMatrixMultiply(x_, y_))) +pyimatmul(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceMatrixMultiply(x_, y_))) """ pyifloordiv(x, y) In-place floor divide. `x = pyifloordiv(x, y)` is equivalent to `x //= y` in Python. """ -pyifloordiv(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceFloorDivide(x_, y_))) +pyifloordiv(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceFloorDivide(x_, y_))) """ pyitruediv(x, y) In-place true division. `x = pyitruediv(x, y)` is equivalent to `x /= y` in Python. """ -pyitruediv(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceTrueDivide(x_, y_))) +pyitruediv(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceTrueDivide(x_, y_))) """ pyimod(x, y) In-place subtraction. `x = pyimod(x, y)` is equivalent to `x %= y` in Python. """ -pyimod(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceRemainder(x_, y_))) +pyimod(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceRemainder(x_, y_))) """ pyilshift(x, y) In-place left shift. `x = pyilshift(x, y)` is equivalent to `x <<= y` in Python. """ -pyilshift(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceLshift(x_, y_))) +pyilshift(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceLshift(x_, y_))) """ pyirshift(x, y) In-place right shift. `x = pyirshift(x, y)` is equivalent to `x >>= y` in Python. """ -pyirshift(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceRshift(x_, y_))) +pyirshift(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceRshift(x_, y_))) """ pyiand(x, y) In-place and. `x = pyiand(x, y)` is equivalent to `x &= y` in Python. """ -pyiand(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceAnd(x_, y_))) +pyiand(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceAnd(x_, y_))) """ pyixor(x, y) In-place xor. `x = pyixor(x, y)` is equivalent to `x ^= y` in Python. """ -pyixor(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceXor(x_, y_))) +pyixor(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceXor(x_, y_))) """ pyior(x, y) In-place or. `x = pyior(x, y)` is equivalent to `x |= y` in Python. """ -pyior(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceOr(x_, y_))) +pyior(x, y) = @pyregion pynew(errcheck(@autopy x y C.PyNumber_InPlaceOr(x_, y_))) # power """ @@ -488,14 +496,14 @@ pyior(x, y) = pynew(errcheck(@autopy x y C.PyNumber_InPlaceOr(x_, y_))) Equivalent to `x ** y` or `pow(x, y, z)` in Python. """ pypow(x, y, z = pybuiltins.None) = - pynew(errcheck(@autopy x y z C.PyNumber_Power(x_, y_, z_))) + @pyregion pynew(errcheck(@autopy x y z C.PyNumber_Power(x_, y_, z_))) """ pyipow(x, y, z=None) In-place power. `x = pyipow(x, y)` is equivalent to `x **= y` in Python. """ pyipow(x, y, z = pybuiltins.None) = - pynew(errcheck(@autopy x y z C.PyNumber_InPlacePower(x_, y_, z_))) + @pyregion pynew(errcheck(@autopy x y z C.PyNumber_InPlacePower(x_, y_, z_))) ### iter @@ -504,7 +512,7 @@ pyipow(x, y, z = pybuiltins.None) = Equivalent to `iter(x)` in Python. """ -pyiter(x) = pynew(errcheck(@autopy x C.PyObject_GetIter(x_))) +pyiter(x) = @pyregion pynew(errcheck(@autopy x C.PyObject_GetIter(x_))) """ pynext(x, [d]) @@ -515,18 +523,22 @@ Returns the next item from the iterator `x`. If there are no more items, returns given, else raises `StopIteration`. """ function pynext(x) - ptr = @pyregion errcheck_ambig(C.PyIter_Next(x)) - if ptr == C.PyNULL - errset(pybuiltins.StopIteration) - pythrow() - else - pynew(ptr) + @pyregion begin + ptr = errcheck_ambig(C.PyIter_Next(x)) + if ptr == C.PyNULL + errset(pybuiltins.StopIteration) + pythrow() + else + pynew(ptr) + end end end function pynext(x, d) - ptr = @pyregion errcheck_ambig(C.PyIter_Next(x)) - ptr == C.PyNULL ? d : pynew(ptr) + @pyregion begin + ptr = errcheck_ambig(C.PyIter_Next(x)) + ptr == C.PyNULL ? d : pynew(ptr) + end end """ @@ -577,7 +589,7 @@ pystr_fromUTF8(x) = pystr_fromUTF8(pointer(x), sizeof(x)) Convert `x` to a Python `str`. """ -pystr(x) = pynew(errcheck(@autopy x C.PyObject_Str(x_))) +pystr(x) = @pyregion pynew(errcheck(@autopy x C.PyObject_Str(x_))) pystr(x::String) = pystr_fromUTF8(x) pystr(x::SubString{String}) = pystr_fromUTF8(x) pystr(x::Char) = pystr(string(x)) @@ -612,7 +624,7 @@ pybytes_fromdata(x) = pybytes_fromdata(pointer(x), sizeof(x)) Convert `x` to a Python `bytes`. """ -pybytes(x) = pynew(errcheck(@autopy x C.PyObject_Bytes(x_))) +pybytes(x) = @pyregion pynew(errcheck(@autopy x C.PyObject_Bytes(x_))) pybytes(x::Vector{UInt8}) = pybytes_fromdata(x) pybytes(x::Base.CodeUnits{UInt8,String}) = pybytes_fromdata(x) pybytes(x::Base.CodeUnits{UInt8,SubString{String}}) = pybytes_fromdata(x) @@ -674,7 +686,7 @@ function pyint(x::Unsigned) end end end -pyint(x) = @autopy x pynew(errcheck(C.PyNumber_Long(x_))) +pyint(x) = @pyregion @autopy x pynew(errcheck(C.PyNumber_Long(x_))) pyisint(x) = pytypecheckfast(x, C.Py_TPFLAGS_LONG_SUBCLASS) @@ -686,11 +698,11 @@ pyisint(x) = pytypecheckfast(x, C.Py_TPFLAGS_LONG_SUBCLASS) Convert `x` to a Python `float`. """ pyfloat(x::Real = 0.0) = @pyregion pynew(errcheck(C.PyFloat_FromDouble(x))) -pyfloat(x) = @autopy x pynew(errcheck(C.PyNumber_Float(x_))) +pyfloat(x) = @pyregion @autopy x pynew(errcheck(C.PyNumber_Float(x_))) pyisfloat(x) = pytypecheck(x, pybuiltins.float) -pyfloat_asdouble(x) = errcheck_ambig(@autopy x C.PyFloat_AsDouble(x_)) +pyfloat_asdouble(x) = @pyregion errcheck_ambig(@autopy x C.PyFloat_AsDouble(x_)) ### complex @@ -709,9 +721,11 @@ pycomplex(x, y) = pybuiltins.complex(x, y) pyiscomplex(x) = pytypecheck(x, pybuiltins.complex) function pycomplex_ascomplex(x) - c = @autopy x C.PyComplex_AsCComplex(x_) - c.real == -1 && c.imag == 0 && errcheck() - return Complex(c.real, c.imag) + @pyregion begin + c = @autopy x C.PyComplex_AsCComplex(x_) + c.real == -1 && c.imag == 0 && errcheck() + return Complex(c.real, c.imag) + end end ### type @@ -721,7 +735,7 @@ end The Python `type` of `x`. """ -pytype(x) = pynew(errcheck(@autopy x C.PyObject_Type(x_))) +pytype(x) = @pyregion pynew(errcheck(@autopy x C.PyObject_Type(x_))) """ pytype(name, bases, dict) @@ -798,8 +812,8 @@ end pyistype(x) = pytypecheckfast(x, C.Py_TPFLAGS_TYPE_SUBCLASS) -pytypecheck(x, t) = (@autopy x t C.Py_TypeCheck(x_, t_)) == 1 -pytypecheckfast(x, f) = (@autopy x C.Py_TypeCheckFast(x_, f)) == 1 +pytypecheck(x, t) = @pyregion (@autopy x t C.Py_TypeCheck(x_, t_)) == 1 +pytypecheckfast(x, f) = @pyregion (@autopy x C.Py_TypeCheckFast(x_, f)) == 1 ### slice @@ -809,7 +823,7 @@ pytypecheckfast(x, f) = (@autopy x C.Py_TypeCheckFast(x_, f)) == 1 Construct a Python `slice`. Unspecified arguments default to `None`. """ pyslice(x, y, z = pybuiltins.None) = - pynew(errcheck(@autopy x y z C.PySlice_New(x_, y_, z_))) + @pyregion pynew(errcheck(@autopy x y z C.PySlice_New(x_, y_, z_))) pyslice(y) = pyslice(pybuiltins.None, y, pybuiltins.None) pyisslice(x) = pytypecheck(x, pybuiltins.slice) @@ -893,9 +907,9 @@ function pylist_setitem(xs::Py, i, x) return xs end -pylist_append(xs::Py, x) = errcheck(@autopy x C.PyList_Append(xs, x_)) +pylist_append(xs::Py, x) = @pyregion errcheck(@autopy x C.PyList_Append(xs, x_)) -pylist_astuple(x) = pynew(errcheck(@autopy x C.PyList_AsTuple(x_))) +pylist_astuple(x) = @pyregion pynew(errcheck(@autopy x C.PyList_AsTuple(x_))) function pylist_fromiter(xs) sz = Base.IteratorSize(typeof(xs)) @@ -965,7 +979,7 @@ end ### set -pyset_add(set::Py, x) = (errcheck(@autopy x C.PySet_Add(set, x_)); set) +pyset_add(set::Py, x) = @pyregion (errcheck(@autopy x C.PySet_Add(set, x_)); set) function pyset_update_fromiter(set::Py, xs) for x in xs @@ -1000,7 +1014,7 @@ pyfrozenset(x) = ispy(x) ? pybuiltins.frozenset(x) : pyfrozenset_fromiter(x) ### dict -pydict_setitem(x::Py, k, v) = errcheck(@autopy k v C.PyDict_SetItem(x, k_, v_)) +pydict_setitem(x::Py, k, v) = @pyregion errcheck(@autopy k v C.PyDict_SetItem(x, k_, v_)) function pydict_fromiter(kvs) ans = pydict() @@ -1065,11 +1079,13 @@ pytime(x::Time) = if iszero(nanosecond(x)) pytime(hour(x), minute(x), second(x), millisecond(x) * 1000 + microsecond(x)) else - errset( - pybuiltins.ValueError, - "cannot create 'datetime.time' with less than microsecond resolution", - ) - pythrow() + @pyregion begin + errset( + pybuiltins.ValueError, + "cannot create 'datetime.time' with less than microsecond resolution", + ) + pythrow() + end end pydatetime( @@ -1483,7 +1499,7 @@ Import a module `m`, or an attribute `k`, or a tuple of attributes. If several arguments are given, return the results of importing each one in a tuple. """ -pyimport(m) = pynew(errcheck(@autopy m C.PyImport_Import(m_))) +pyimport(m) = @pyregion pynew(errcheck(@autopy m C.PyImport_Import(m_))) pyimport((m, k)::Pair) = (m_ = pyimport(m); k_ = pygetattr(m_, k); unsafe_pydel(m_); k_) pyimport((m, ks)::Pair{<:Any,<:Tuple}) = (m_ = pyimport(m); ks_ = map(k -> pygetattr(m_, k), ks); unsafe_pydel(m_); ks_) diff --git a/src/Core/err.jl b/src/Core/err.jl index 8e49ab44..5a83dd06 100644 --- a/src/Core/err.jl +++ b/src/Core/err.jl @@ -3,7 +3,7 @@ errval(::T) where {T<:Number} = zero(T) - one(T) iserrval(val) = val == errval(val) -iserrset() = @pyregion C.PyErr_Occurred() != C.PyNULL +iserrset() = C.PyErr_Occurred() != C.PyNULL iserrset(val) = val == errval(val) errcheck() = iserrset() ? pythrow() : nothing @@ -13,35 +13,31 @@ iserrset_ambig(val) = iserrset(val) && iserrset() errcheck_ambig(val) = iserrset_ambig(val) ? pythrow() : val -errclear() = @pyregion C.PyErr_Clear() +errclear() = C.PyErr_Clear() -errmatches(t) = (@autopy t C.PyErr_ExceptionMatches(t_)) == 1 +errmatches(t) = C.PyErr_ExceptionMatches(Py(t)) == 1 function errget() - @pyregion begin - t = Ref(C.PyNULL) - v = Ref(C.PyNULL) - b = Ref(C.PyNULL) - C.PyErr_Fetch(t, v, b) - (pynew(t[]), pynew(v[]), pynew(b[])) - end + t = Ref(C.PyNULL) + v = Ref(C.PyNULL) + b = Ref(C.PyNULL) + C.PyErr_Fetch(t, v, b) + (pynew(t[]), pynew(v[]), pynew(b[])) end -errset(t::Py) = @pyregion C.PyErr_SetNone(t) -errset(t::Py, v::Py) = @pyregion C.PyErr_SetObject(t, v) -errset(t::Py, v::String) = @pyregion C.PyErr_SetString(t, v) +errset(t::Py) = C.PyErr_SetNone(t) +errset(t::Py, v::Py) = C.PyErr_SetObject(t, v) +errset(t::Py, v::String) = C.PyErr_SetString(t, v) function errnormalize!(t::Py, v::Py, b::Py) - @pyregion begin - tref = Ref(getptr(t)) - vref = Ref(getptr(v)) - bref = Ref(getptr(b)) - C.PyErr_NormalizeException(tref, vref, bref) - setptr!(t, tref[]) - setptr!(v, vref[]) - setptr!(b, bref[]) - (t, v, b) - end + tref = Ref(getptr(t)) + vref = Ref(getptr(v)) + bref = Ref(getptr(b)) + C.PyErr_NormalizeException(tref, vref, bref) + setptr!(t, tref[]) + setptr!(v, vref[]) + setptr!(b, bref[]) + (t, v, b) end function PyException(v::Py = pybuiltins.None) @@ -67,15 +63,17 @@ function Base.show(io::IO, x::PyException) end function Base.getproperty(exc::PyException, k::Symbol) - if k in (:t, :v, :b) && !exc._isnormalized - errnormalize!(exc._t, exc._v, exc._b) - pyisnull(exc._t) && pycopy!(exc._t, pybuiltins.None) - pyisnull(exc._v) && pycopy!(exc._v, pybuiltins.None) - pyisnull(exc._b) && pycopy!(exc._b, pybuiltins.None) - pyisnone(exc._v) || (exc._v.__traceback__ = exc._b) - exc._isnormalized = true + @pyregion begin + if k in (:t, :v, :b) && !exc._isnormalized + errnormalize!(exc._t, exc._v, exc._b) + pyisnull(exc._t) && pycopy!(exc._t, pybuiltins.None) + pyisnull(exc._v) && pycopy!(exc._v, pybuiltins.None) + pyisnull(exc._b) && pycopy!(exc._b, pybuiltins.None) + pyisnone(exc._v) || (exc._v.__traceback__ = exc._b) + exc._isnormalized = true + end + k == :t ? exc._t : k == :v ? exc._v : k == :b ? exc._b : getfield(exc, k) end - k == :t ? exc._t : k == :v ? exc._v : k == :b ? exc._b : getfield(exc, k) end pythrow() = throw(PyException(errget()..., false)) diff --git a/src/JlWrap/any.jl b/src/JlWrap/any.jl index e58c2aa6..d65cb597 100644 --- a/src/JlWrap/any.jl +++ b/src/JlWrap/any.jl @@ -298,7 +298,7 @@ function pyjlany_index(self) if self isa Integer pyint(self) else - errset( + @pyregion errset( pybuiltins.TypeError, "Only Julia 'Integer' values can be used as Python indices, not '$(typeof(self))'", ) @@ -310,7 +310,7 @@ function pyjlany_bool(self) if self isa Bool pybool(self) else - errset( + @pyregion errset( pybuiltins.TypeError, "Only Julia 'Bool' values can be tested for truthyness, not '$(typeof(self))'", ) @@ -370,7 +370,7 @@ end function pyjlany_next(self) s = iterate(self) if s === nothing - errset(pybuiltins.StopIteration) + @pyregion errset(pybuiltins.StopIteration) PyNULL else pyjl(s[1]) @@ -380,7 +380,7 @@ end function pyjliter_next(self) s = iterate(self) if s === nothing - errset(pybuiltins.StopIteration) + @pyregion errset(pybuiltins.StopIteration) PyNULL else Py(s[1]) @@ -427,7 +427,7 @@ function pyjlany_numpy_dtype(self::Type) ) end if pyisnull(ans) - errset(pybuiltins.AttributeError, "__numpy_dtype__") + @pyregion errset(pybuiltins.AttributeError, "__numpy_dtype__") end return ans end diff --git a/src/JlWrap/array.jl b/src/JlWrap/array.jl index ebe4a272..dc39da39 100644 --- a/src/JlWrap/array.jl +++ b/src/JlWrap/array.jl @@ -3,16 +3,16 @@ const pyjlarraytype = pynew() function pyjl_getaxisindex(x::AbstractUnitRange{<:Integer}, k::Py) if pyisslice(k) a = @pyconvert Union{Int,Nothing} k.start begin - errset(pybuiltins.TypeError, "slice components must be integers") - pythrow() + @pyregion errset(pybuiltins.TypeError, "slice components must be integers") + @pyregion pythrow() end b = @pyconvert Union{Int,Nothing} k.step begin - errset(pybuiltins.TypeError, "slice components must be integers") - pythrow() + @pyregion errset(pybuiltins.TypeError, "slice components must be integers") + @pyregion pythrow() end c = @pyconvert Union{Int,Nothing} k.stop begin - errset(pybuiltins.TypeError, "slice components must be integers") - pythrow() + @pyregion errset(pybuiltins.TypeError, "slice components must be integers") + @pyregion pythrow() end # step defaults to 1 b′ = b === nothing ? 1 : b @@ -26,8 +26,8 @@ function pyjl_getaxisindex(x::AbstractUnitRange{<:Integer}, k::Py) a′ = Int(last(x)) c′ = Int(first(x)) else - errset(pybuiltins.ValueError, "step must be non-zero") - pythrow() + @pyregion errset(pybuiltins.ValueError, "step must be non-zero") + @pyregion pythrow() end else # start defaults @@ -41,23 +41,23 @@ function pyjl_getaxisindex(x::AbstractUnitRange{<:Integer}, k::Py) if checkbounds(Bool, x, r) return r else - errset(pybuiltins.IndexError, "array index out of bounds") - pythrow() + @pyregion errset(pybuiltins.IndexError, "array index out of bounds") + @pyregion pythrow() end else j = @pyconvert Int k begin - errset( + @pyregion errset( pybuiltins.TypeError, "index must be slice or integer, got '$(pytype(k).__name__)'", ) - pythrow() + @pyregion pythrow() end r = Int(j < 0 ? (last(x) + j + 1) : (first(x) + j)) if checkbounds(Bool, x, r) return r else - errset(pybuiltins.IndexError, "array index out of bounds") - pythrow() + @pyregion errset(pybuiltins.IndexError, "array index out of bounds") + @pyregion pythrow() end end end @@ -72,13 +72,13 @@ function pyjl_getarrayindices(x::AbstractArray{T,N}, ks::Py) where {T,N} return ans end else - errset(pybuiltins.TypeError, "expecting $N indices, got $(pylen(ks))") - pythrow() + @pyregion errset(pybuiltins.TypeError, "expecting $N indices, got $(pylen(ks))") + @pyregion pythrow() end elseif N == 1 return (pyjl_getaxisindex(axes(x, 1), ks),) else - errset(pybuiltins.TypeError, "expecting $N indices, got 1") + @pyregion errset(pybuiltins.TypeError, "expecting $N indices, got 1") end end @@ -111,8 +111,8 @@ function pyjlarray_delitem(x::AbstractArray{T,N}, k_::Py) where {T,N} unsafe_pydel(k_) deleteat!(x, k...) else - errset(pybuiltins.TypeError, "can only delete from 1D arrays") - pythrow() + @pyregion errset(pybuiltins.TypeError, "can only delete from 1D arrays") + @pyregion pythrow() end return Py(nothing) end @@ -320,7 +320,7 @@ function pyjlarray_array_interface(x::AbstractArray{T,N}) where {T,N} return d end end - errset(pybuiltins.AttributeError, "__array_interface__") + @pyregion errset(pybuiltins.AttributeError, "__array_interface__") return PyNULL end pyjl_handle_error_type(::typeof(pyjlarray_array_interface), x, exc) = diff --git a/src/JlWrap/base.jl b/src/JlWrap/base.jl index 67723e59..29c8ec36 100644 --- a/src/JlWrap/base.jl +++ b/src/JlWrap/base.jl @@ -4,7 +4,7 @@ _pyjl_getvalue(x) = @autopy x Cjl.PyJuliaValue_GetValue(x_) _pyjl_setvalue!(x, v) = @autopy x Cjl.PyJuliaValue_SetValue(x_, v) -pyjl(t, v) = pynew(errcheck(@autopy t Cjl.PyJuliaValue_New(t_, v))) +pyjl(t, v) = @pyregion pynew(errcheck(@autopy t Cjl.PyJuliaValue_New(t_, v))) """ pyisjl(x) @@ -34,57 +34,59 @@ pyconvert_rule_jlvalue(::Type{T}, x::Py) where {T} = function Cjl._pyjl_callmethod(f, self_::C.PyPtr, args_::C.PyPtr, nargs::C.Py_ssize_t) @nospecialize f - in_f = false - self = Cjl.PyJuliaValue_GetValue(self_) - try - if nargs == 1 - in_f = true - ans = @pyregionbreak(f(self))::Py - in_f = false - elseif nargs == 2 - arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) - in_f = true - ans = @pyregionbreak(f(self, arg1))::Py - in_f = false - elseif nargs == 3 - arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) - arg2 = pynew(incref(C.PyTuple_GetItem(args_, 2))) - in_f = true - ans = @pyregionbreak(f(self, arg1, arg2))::Py - in_f = false - elseif nargs == 4 - arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) - arg2 = pynew(incref(C.PyTuple_GetItem(args_, 2))) - arg3 = pynew(incref(C.PyTuple_GetItem(args_, 3))) - in_f = true - ans = @pyregionbreak(f(self, arg1, arg2, arg3))::Py - in_f = false - else - errset( - pybuiltins.NotImplementedError, - "__jl_callmethod not implemented for this many arguments", - ) - end - return getptr(incref(ans)) - catch exc - if exc isa PyException - Base.GC.@preserve exc C.PyErr_Restore( - incref(exc._t), - incref(exc._v), - incref(exc._b), - ) - return C.PyNULL - else - try - if in_f - return pyjl_handle_error(f, self, exc) - else - errset(pyJuliaError, pytuple((pyjl(exc), pyjl(catch_backtrace())))) + @pyregion begin + in_f = false + self = Cjl.PyJuliaValue_GetValue(self_) + try + if nargs == 1 + in_f = true + ans = @pyregionbreak(f(self))::Py + in_f = false + elseif nargs == 2 + arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) + in_f = true + ans = @pyregionbreak(f(self, arg1))::Py + in_f = false + elseif nargs == 3 + arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) + arg2 = pynew(incref(C.PyTuple_GetItem(args_, 2))) + in_f = true + ans = @pyregionbreak(f(self, arg1, arg2))::Py + in_f = false + elseif nargs == 4 + arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) + arg2 = pynew(incref(C.PyTuple_GetItem(args_, 2))) + arg3 = pynew(incref(C.PyTuple_GetItem(args_, 3))) + in_f = true + ans = @pyregionbreak(f(self, arg1, arg2, arg3))::Py + in_f = false + else + @pyregion errset( + pybuiltins.NotImplementedError, + "__jl_callmethod not implemented for this many arguments", + ) + end + return getptr(incref(ans)) + catch exc + if exc isa PyException + Base.GC.@preserve exc C.PyErr_Restore( + incref(exc._t), + incref(exc._v), + incref(exc._b), + ) + return C.PyNULL + else + try + if in_f + return pyjl_handle_error(f, self, exc) + else + @pyregion errset(pyJuliaError, pytuple((pyjl(exc), pyjl(catch_backtrace())))) + return C.PyNULL + end + catch + @pyregion errset(pyJuliaError, "an error occurred while setting an error") return C.PyNULL end - catch - errset(pyJuliaError, "an error occurred while setting an error") - return C.PyNULL end end end @@ -95,11 +97,11 @@ function pyjl_handle_error(f, self, exc) t = pyjl_handle_error_type(f, self, exc)::Py if pyisnull(t) # NULL => raise JuliaError - errset(pyJuliaError, pytuple((pyjl(exc), pyjl(catch_backtrace())))) + @pyregion errset(pyJuliaError, pytuple((pyjl(exc), pyjl(catch_backtrace())))) return C.PyNULL elseif pyistype(t) # Exception type => raise this type of error - errset(t, string("Julia: ", Py(sprint(showerror, exc)))) + @pyregion errset(t, string("Julia: ", Py(sprint(showerror, exc)))) return C.PyNULL else # Otherwise, return the given object (e.g. NotImplemented) diff --git a/src/JlWrap/io.jl b/src/JlWrap/io.jl index 0f7c07d6..ed47400f 100644 --- a/src/JlWrap/io.jl +++ b/src/JlWrap/io.jl @@ -35,7 +35,7 @@ function pyjlio_seek(io::IO, offset_::Py, whence_::Py) seekend(io) pos = position(io) + offset else - errset(pybuiltins.ValueError, "Argument 'whence' must be 0, 1 or 2") + @pyregion errset(pybuiltins.ValueError, "Argument 'whence' must be 0, 1 or 2") return PyNULL end seek(io, pos) @@ -98,14 +98,14 @@ function pyjlbinaryio_readinto(io::IO, b::Py) c = m.c_contiguous if !pytruth(c) unsafe_pydel(c) - errset(pybuiltins.ValueError, "input buffer is not contiguous") + @pyregion errset(pybuiltins.ValueError, "input buffer is not contiguous") return PyNULL end unsafe_pydel(c) buf = unsafe_load(C.PyMemoryView_GET_BUFFER(m)) if buf.readonly != 0 unsafe_pydel(m) - errset(pybuiltins.ValueError, "output buffer is read-only") + @pyregion errset(pybuiltins.ValueError, "output buffer is read-only") return PyNULL end data = unsafe_wrap(Array, Ptr{UInt8}(buf.buf), buf.len) @@ -121,7 +121,7 @@ function pyjlbinaryio_write(io::IO, b::Py) c = m.c_contiguous if !pytruth(c) unsafe_pydel(c) - errset(pybuiltins.ValueError, "input buffer is not contiguous") + @pyregion errset(pybuiltins.ValueError, "input buffer is not contiguous") return PyNULL end unsafe_pydel(c) @@ -212,7 +212,7 @@ function pyjltextio_write(io::IO, s_::Py) # TODO: is this the number of source characters, or the number of output characters? Py(length(s)) else - errset( + @pyregion errset( pybuiltins.TypeError, "Argument 's' must be a 'str', got a '$(pytype(s_).__name__)'", ) diff --git a/src/JlWrap/set.jl b/src/JlWrap/set.jl index 36f19f02..9bf5e3f8 100644 --- a/src/JlWrap/set.jl +++ b/src/JlWrap/set.jl @@ -11,7 +11,7 @@ end function pyjlset_pop(x::AbstractSet) if isempty(x) - errset(pybuiltins.KeyError, "pop from an empty set") + @pyregion errset(pybuiltins.KeyError, "pop from an empty set") PyNULL else Py(pop!(x)) @@ -20,14 +20,14 @@ end function pyjlset_remove(x::AbstractSet, v_::Py) v = @pyconvert eltype(x) v_ begin - errset(pybuiltins.KeyError, v_) + @pyregion errset(pybuiltins.KeyError, v_) return PyNULL end if v in x delete!(x, v) return Py(nothing) else - errset(pybuiltins.KeyError, v_) + @pyregion errset(pybuiltins.KeyError, v_) return PyNULL end end diff --git a/src/JlWrap/vector.jl b/src/JlWrap/vector.jl index 8058098c..0e39effc 100644 --- a/src/JlWrap/vector.jl +++ b/src/JlWrap/vector.jl @@ -39,7 +39,7 @@ function pyjlvector_insert(x::AbstractVector, k_::Py, v_::Py) insert!(x, k′, v) return Py(nothing) else - errset(pybuiltins.IndexError, "array index out of bounds") + @pyregion errset(pybuiltins.IndexError, "array index out of bounds") return PyNULL end end @@ -75,19 +75,19 @@ function pyjlvector_pop(x::AbstractVector, k_::Py) end return Py(v) else - errset(pybuiltins.IndexError, "pop from empty array") + @pyregion errset(pybuiltins.IndexError, "pop from empty array") return PyNULL end end function pyjlvector_remove(x::AbstractVector, v_::Py) v = @pyconvert eltype(x) v_ begin - errset(pybuiltins.ValueError, "value not in array") + @pyregion errset(pybuiltins.ValueError, "value not in array") return PyNULL end k = findfirst(==(v), x) if k === nothing - errset(pybuiltins.ValueError, "value not in array") + @pyregion errset(pybuiltins.ValueError, "value not in array") return PyNULL end deleteat!(x, k) @@ -96,12 +96,12 @@ end function pyjlvector_index(x::AbstractVector, v_::Py) v = @pyconvert eltype(x) v_ begin - errset(pybuiltins.ValueError, "value not in array") + @pyregion errset(pybuiltins.ValueError, "value not in array") return PyNULL end k = findfirst(==(v), x) if k === nothing - errset(pybuiltins.ValueError, "value not in array") + @pyregion errset(pybuiltins.ValueError, "value not in array") return PyNULL end Py(k - first(axes(x, 1))) From 61e51b1d3026601a72f221a6b32f26be4244710a Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Sat, 3 Oct 2026 14:57:07 +0100 Subject: [PATCH 4/7] Address thread-state review feedback --- docs/src/juliacall.md | 10 +-- docs/src/pythoncall-reference.md | 6 -- src/API/publics.jl | 1 - src/C/C.jl | 8 +- src/C/context.jl | 21 +++-- src/{Region/Region.jl => C/regions.jl} | 103 ++++++++------------- src/Convert/pyconvert.jl | 36 +++----- src/Convert/rules.jl | 9 +- src/Core/Py.jl | 18 ++-- src/Core/err.jl | 6 +- src/GC/GC.jl | 7 +- src/GIL/GIL.jl | 118 ------------------------- src/JlWrap/C.jl | 2 +- src/JlWrap/JlWrap.jl | 1 - src/JlWrap/array.jl | 60 ++++++++----- src/JlWrap/base.jl | 100 ++++++++++----------- src/PyMacro/PyMacro.jl | 18 +--- src/PythonCall.jl | 6 -- src/Utils/Utils.jl | 57 ++++++------ test/Core.jl | 4 +- test/GC.jl | 4 +- test/GIL.jl | 12 +-- 22 files changed, 220 insertions(+), 387 deletions(-) rename src/{Region/Region.jl => C/regions.jl} (67%) delete mode 100644 src/GIL/GIL.jl diff --git a/docs/src/juliacall.md b/docs/src/juliacall.md index 8deb7f07..724c3c4b 100644 --- a/docs/src/juliacall.md +++ b/docs/src/juliacall.md @@ -155,25 +155,19 @@ be configured in two ways: From v0.9.22, JuliaCall supports multi-threading in Julia and/or Python, with some caveats. -Most importantly, you can only call Python code while Python's -[Global Interpreter Lock (GIL)](https://docs.python.org/3/glossary.html#term-global-interpreter-lock) JuliaCall borrows the Python thread state which entered Julia and automatically detaches it while arbitrary Julia code runs. Nested Python interaction from that Julia code temporarily reattaches the same state, and the borrowed state is restored before returning to Python. -The historical `_jl_call_nogil` spelling remains available as a compatibility alias: - ```python from concurrent.futures import ThreadPoolExecutor, wait from juliacall import Main as jl pool = ThreadPoolExecutor(4) -fs = [pool.submit(jl.Libc.systemsleep._jl_call_nogil, 5) for _ in range(4)] +fs = [pool.submit(jl.Libc.systemsleep, 5) for _ in range(4)] wait(fs) ``` -Ordinary calls provide the same automatic resource management, so -`pool.submit(jl.Libc.systemsleep, 5)` is preferred. PythonCall operations nested inside -Julia callbacks are safe without explicit region or lock calls. +PythonCall operations nested inside Julia callbacks are safe without explicit region calls. You can also use [multi-threading from Julia](@ref jl-multi-threading). diff --git a/docs/src/pythoncall-reference.md b/docs/src/pythoncall-reference.md index 9b6400f5..7f3e6620 100644 --- a/docs/src/pythoncall-reference.md +++ b/docs/src/pythoncall-reference.md @@ -230,13 +230,7 @@ performance and concurrency hints; users normally do not need them for correctne @pyregionbreak ``` -The older `PythonCall.GIL` names remain as compatibility aliases. - ```@docs -PythonCall.GIL.lock -PythonCall.GIL.@lock -PythonCall.GIL.unlock -PythonCall.GIL.@unlock PythonCall.GC.gc ``` diff --git a/src/API/publics.jl b/src/API/publics.jl index bda0fc47..965f959e 100644 --- a/src/API/publics.jl +++ b/src/API/publics.jl @@ -2,7 +2,6 @@ if Base.VERSION ≥ v"1.11" eval(Meta.parse(""" public GC, - GIL, VERSION, # C diff --git a/src/C/C.jl b/src/C/C.jl index f7b4e020..437232c9 100644 --- a/src/C/C.jl +++ b/src/C/C.jl @@ -19,11 +19,17 @@ if @load_preference("exe", "@CondaPkg") == "@CondaPkg" end import ..PythonCall: - python_executable_path, python_library_path, python_library_handle, python_version + @pyregion, + @pyregionbreak, + python_executable_path, + python_library_path, + python_library_handle, + python_version include("consts.jl") include("pointers.jl") include("extras.jl") +include("regions.jl") include("context.jl") include("api.jl") diff --git a/src/C/context.jl b/src/C/context.jl index 220bf88d..c9cf976d 100644 --- a/src/C/context.jl +++ b/src/C/context.jl @@ -18,14 +18,10 @@ A handle to a loaded instance of libpython, its interpreter, function pointers, which::Symbol = :unknown # :CondaPkg, :PyCall, :embedded or :unknown version::Union{VersionNumber,Missing} = missing is_free_threaded::Bool = false + interp::Ptr{Cvoid} = C_NULL end const CTX = Context() -const FINALIZE_HOOK = Ref{Function}(() -> begin - if Py_FinalizeEx() == -1 - @warn "Py_FinalizeEx() error" - end -end) function _atpyexit() if CTX.is_initialized && !CTX.is_preinitialized @@ -287,7 +283,12 @@ function init_context() Py_InitializeEx(0) atexit() do CTX.is_initialized = false - FINALIZE_HOOK[]() + if current_tstate() == C_NULL + ts = THREAD_STATE() + ts.tstate == C_NULL && (ts.tstate = PyThreadState_New(CTX.interp)) + PyEval_RestoreThread(ts.tstate) + end + Py_FinalizeEx() == -1 && @warn "Py_FinalizeEx() error" end end CTX.is_initialized = true @@ -321,6 +322,14 @@ function init_context() ) CTX.is_free_threaded = occursin("free-threading build", verstr) + current = current_tstate() + current == C_NULL && error("Python initialization did not leave an attached thread state") + CTX.interp = PyThreadState_GetInterpreter(current) + CTX.interp == C_NULL && error("could not determine the Python interpreter") + if !CTX.is_embedded && !CTX.is_preinitialized + PyEval_SaveThread() + end + launch_on_main_thread(Threads.threadid()) # makes on_main_thread usable @debug "Initialized PythonCall.jl" CTX.is_embedded CTX.is_initialized CTX.exe_path CTX.lib_path CTX.lib_ptr CTX.pyprogname CTX.pyhome CTX.version CTX.is_free_threaded diff --git a/src/Region/Region.jl b/src/C/regions.jl similarity index 67% rename from src/Region/Region.jl rename to src/C/regions.jl index d69e1fab..26af46fe 100644 --- a/src/Region/Region.jl +++ b/src/C/regions.jl @@ -1,10 +1,3 @@ -"""Internal task-safe management of CPython thread states.""" -module Region - -using ..C: C -using ..Utils -import ..PythonCall: @pyregion, @pyregionbreak - const PyThreadStatePtr = Ptr{Cvoid} mutable struct ThreadState @@ -24,10 +17,9 @@ TaskState() = TaskState(C_NULL, nothing, false, 0, false) const THREAD_STATE = Utils.OncePerThread{ThreadState}(ThreadState) const TASK_STATE = Utils.OncePerTask{TaskState}(TaskState) -const INTERP = Ref{Ptr{Cvoid}}(C_NULL) -current_tstate() = C.PyThreadState_GetUnchecked() -has_tstate() = C.CTX.is_initialized && current_tstate() != C_NULL +current_tstate() = PyThreadState_GetUnchecked() +has_tstate() = CTX.is_initialized && current_tstate() != C_NULL function reset!(s::TaskState) s.tstate = C_NULL @@ -37,8 +29,7 @@ function reset!(s::TaskState) return end -function start_session!(s::TaskState) - task = current_task() +function start_session!(task::Task, s::TaskState) s.oldsticky = task.sticky task.sticky = true s.tid = Threads.threadid() @@ -47,18 +38,20 @@ function start_session!(s::TaskState) return ts end -check_thread(s::TaskState) = (@assert current_task().sticky && Threads.threadid() == s.tid) +function check_thread(task::Task, s::TaskState) + @assert task.sticky + @assert Threads.threadid() == s.tid +end -function enter_region() - s = TASK_STATE() +function enter_region(task::Task, s::TaskState) if s.tstate != C_NULL - check_thread(s) + check_thread(task, s) if s.attached return :noop end Base.acquire(s.sem::Base.Semaphore) try - C.PyEval_RestoreThread(s.tstate) + PyEval_RestoreThread(s.tstate) s.attached = true catch Base.release(s.sem::Base.Semaphore) @@ -67,9 +60,8 @@ function enter_region() return :detach end - task = current_task() ts = try - start_session!(s) + start_session!(task, s) catch task.sticky = s.oldsticky reset!(s) @@ -86,11 +78,11 @@ function enter_region() return :root_borrowed end if ts.tstate == C_NULL - ts.tstate = C.PyThreadState_New(INTERP[]) + ts.tstate = PyThreadState_New(CTX.interp) ts.tstate == C_NULL && error("PyThreadState_New failed") end s.tstate = ts.tstate - C.PyEval_RestoreThread(s.tstate) + PyEval_RestoreThread(s.tstate) s.attached = true return :root_owned catch @@ -101,40 +93,38 @@ function enter_region() end end -function exit_region(token) +function exit_region(task::Task, s::TaskState, token) token === :noop && return - s = TASK_STATE() - check_thread(s) + check_thread(task, s) if token === :detach - saved = C.PyEval_SaveThread() + saved = PyEval_SaveThread() @assert saved == s.tstate s.attached = false Base.release(s.sem::Base.Semaphore) elseif token === :root_owned - saved = C.PyEval_SaveThread() + saved = PyEval_SaveThread() @assert saved == s.tstate sem, oldsticky = s.sem::Base.Semaphore, s.oldsticky reset!(s) Base.release(sem) - current_task().sticky = oldsticky + task.sticky = oldsticky elseif token === :root_borrowed @assert current_tstate() == s.tstate sem, oldsticky = s.sem::Base.Semaphore, s.oldsticky reset!(s) Base.release(sem) - current_task().sticky = oldsticky + task.sticky = oldsticky else error("invalid Python region token") end return end -function enter_break() - s = TASK_STATE() +function enter_break(task::Task, s::TaskState) if s.tstate != C_NULL - check_thread(s) + check_thread(task, s) !s.attached && return :noop - saved = C.PyEval_SaveThread() + saved = PyEval_SaveThread() @assert saved == s.tstate s.attached = false Base.release(s.sem::Base.Semaphore) @@ -142,9 +132,8 @@ function enter_break() end current_tstate() == C_NULL && return :noop - task = current_task() ts = try - start_session!(s) + start_session!(task, s) catch task.sticky = s.oldsticky reset!(s) @@ -162,7 +151,7 @@ function enter_break() return :noop end s.tstate = current - saved = C.PyEval_SaveThread() + saved = PyEval_SaveThread() @assert saved == current s.attached = false Base.release(ts.sem) @@ -175,18 +164,17 @@ function enter_break() end end -function exit_break(token) +function exit_break(task::Task, s::TaskState, token) token === :noop && return - s = TASK_STATE() - check_thread(s) + check_thread(task, s) Base.acquire(s.sem::Base.Semaphore) - C.PyEval_RestoreThread(s.tstate) + PyEval_RestoreThread(s.tstate) s.attached = true if token === :root_restore sem, oldsticky = s.sem::Base.Semaphore, s.oldsticky reset!(s) Base.release(sem) - current_task().sticky = oldsticky + task.sticky = oldsticky elseif token !== :restore error("invalid Python region-break token") end @@ -203,11 +191,13 @@ blocks cooperatively in [`@pyregionbreak`](@ref). """ macro pyregion(ex) quote - local token = $enter_region() + local task = current_task() + local state = $TASK_STATE() + local token = $enter_region(task, state) try $(esc(ex)) finally - $exit_region(token) + $exit_region(task, state, token) end end end @@ -221,34 +211,13 @@ PythonCall operations still work automatically, and both kinds of region nest fr """ macro pyregionbreak(ex) quote - local token = $enter_break() + local task = current_task() + local state = $TASK_STATE() + local token = $enter_break(task, state) try $(esc(ex)) finally - $exit_break(token) + $exit_break(task, state, token) end end end - -function __init__() - current = current_tstate() - current == C_NULL && error("Python initialization did not leave an attached thread state") - INTERP[] = C.PyThreadState_GetInterpreter(current) - INTERP[] == C_NULL && error("could not determine the Python interpreter") - return -end - -function start_runtime() - if !C.CTX.is_embedded && !C.CTX.is_preinitialized - C.PyEval_SaveThread() - C.FINALIZE_HOOK[] = function () - ts = THREAD_STATE() - ts.tstate == C_NULL && (ts.tstate = C.PyThreadState_New(INTERP[])) - C.PyEval_RestoreThread(ts.tstate) - C.Py_FinalizeEx() == -1 && @warn "Py_FinalizeEx() error" - end - end - return -end - -end diff --git a/src/Convert/pyconvert.jl b/src/Convert/pyconvert.jl index ab69aad9..3df5f7f6 100644 --- a/src/Convert/pyconvert.jl +++ b/src/Convert/pyconvert.jl @@ -353,7 +353,8 @@ function pytryconvert(::Type{T}, x_) where {T} # apply the rules for rule in rules - ans2 = rule(x)::pyconvert_returntype(T) + # Conversion rules are user-extensible Julia code and may yield. + ans2 = @pyregionbreak(rule(x))::pyconvert_returntype(T) pyconvert_isunconverted(ans2) || return ans2 end @@ -389,19 +390,10 @@ Convert the Python object `x` to a `T`. If `d` is specified, it is returned on failure instead of throwing an error. """ -function pyconvert(::Type{T}, x) where {T} - @pyregion begin - @autopy x @pyconvert T x_ error( - "cannot convert this Python '$(pytype(x_).__name__)' to a Julia '$T'", - ) - end -end - -function pyconvert(::Type{T}, x, d) where {T} - @pyregion begin - @autopy x @pyconvert T x_ d - end -end +pyconvert(::Type{T}, x) where {T} = @autopy x @pyconvert T x_ error( + "cannot convert this Python '$(pytype(x_).__name__)' to a Julia '$T'", +) +pyconvert(::Type{T}, x, d) where {T} = @autopy x @pyconvert T x_ d """ pyconvertarg(T, x, name) @@ -410,16 +402,12 @@ Convert the Python object `x` to a `T`. On failure, throws a Python `TypeError` saying that the argument `name` could not be converted. """ -function pyconvertarg(::Type{T}, x, name) where {T} - @pyregion begin - @autopy x @pyconvert T x_ begin - errset( - pybuiltins.TypeError, - "Cannot convert argument '$name' to a Julia '$T', got a '$(pytype(x_).__name__)'", - ) - pythrow() - end - end +pyconvertarg(::Type{T}, x, name) where {T} = @autopy x @pyconvert T x_ begin + errset( + pybuiltins.TypeError, + "Cannot convert argument '$name' to a Julia '$T', got a '$(pytype(x_).__name__)'", + ) + pythrow() end function init_pyconvert() diff --git a/src/Convert/rules.jl b/src/Convert/rules.jl index 43e029b8..7d514c7f 100644 --- a/src/Convert/rules.jl +++ b/src/Convert/rules.jl @@ -58,11 +58,10 @@ pyconvert_rule_bytes(::Type{Base.CodeUnits{UInt8,String}}, x::Py) = ### int -pyconvert_rule_int(::Type{T}, x::Py) where {T<:Number} = begin - # first try to convert to Clonglong (or Culonglong if unsigned) - v = - T <: Unsigned ? C.PyLong_AsUnsignedLongLong(x) : - C.PyLong_AsLongLong(x) +pyconvert_rule_int(::Type{T}, x::Py) where {T<:Number} = @pyregion begin + # First try to convert to Clonglong (or Culonglong if unsigned). Conversion rules + # run outside their caller's region because user-defined rules may yield. + v = T <: Unsigned ? C.PyLong_AsUnsignedLongLong(x) : C.PyLong_AsLongLong(x) if !iserrset_ambig(v) # success return pyconvert_tryconvert(T, v) diff --git a/src/Core/Py.jl b/src/Core/Py.jl index f86a1670..b769a250 100644 --- a/src/Core/Py.jl +++ b/src/Core/Py.jl @@ -1,5 +1,5 @@ -incref(x::C.PyPtr) = @pyregion (C.Py_IncRef(x); x) -decref(x::C.PyPtr) = @pyregion (C.Py_DecRef(x); x) +incref(x::C.PyPtr) = (C.Py_IncRef(x); x) +decref(x::C.PyPtr) = (C.Py_DecRef(x); x) """ ispy(x) @@ -85,12 +85,10 @@ Use this to eagerly free a Python object, rather than waiting for Julia's GC to it at some indeterminate point in the future. """ function unsafe_pydel(x::Py) - @pyregion begin - ptr = getptr(x) - if ptr != C.PyNULL - C.Py_DecRef(ptr) - setptr!(x, C.PyNULL) - end + ptr = getptr(x) + if ptr != C.PyNULL + @pyregion C.Py_DecRef(ptr) + setptr!(x, C.PyNULL) end return end @@ -274,7 +272,7 @@ Base.hasproperty(x::Py, k::String) = pyhasattr(x, k) Base.setproperty!(x::Py, k::Symbol, v) = pysetattr(x, string(k), v) Base.setproperty!(x::Py, k::String, v) = pysetattr(x, k, v) -function _propertynames(x::Py, private::Bool) +function Base.propertynames(x::Py, private::Bool = false) # this follows the logic of rlcompleter.py function classmembers(c) r = pydir(c) @@ -295,8 +293,6 @@ function _propertynames(x::Py, private::Bool) return Symbol[Symbol(pystr_asstring(word)) for word in words] end -Base.propertynames(x::Py, private::Bool = false) = @pyregion _propertynames(x, private) - Base.Bool(x::Py) = pytruth(x) Base.length(x::Py) = pylen(x) diff --git a/src/Core/err.jl b/src/Core/err.jl index 5a83dd06..becce1f7 100644 --- a/src/Core/err.jl +++ b/src/Core/err.jl @@ -63,8 +63,8 @@ function Base.show(io::IO, x::PyException) end function Base.getproperty(exc::PyException, k::Symbol) - @pyregion begin - if k in (:t, :v, :b) && !exc._isnormalized + if k in (:t, :v, :b) && !exc._isnormalized + @pyregion begin errnormalize!(exc._t, exc._v, exc._b) pyisnull(exc._t) && pycopy!(exc._t, pybuiltins.None) pyisnull(exc._v) && pycopy!(exc._v, pybuiltins.None) @@ -72,8 +72,8 @@ function Base.getproperty(exc::PyException, k::Symbol) pyisnone(exc._v) || (exc._v.__traceback__ = exc._b) exc._isnormalized = true end - k == :t ? exc._t : k == :v ? exc._v : k == :b ? exc._b : getfield(exc, k) end + k == :t ? exc._t : k == :v ? exc._v : k == :b ? exc._b : getfield(exc, k) end pythrow() = throw(PyException(errget()..., false)) diff --git a/src/GC/GC.jl b/src/GC/GC.jl index 5655bf6a..f438d572 100644 --- a/src/GC/GC.jl +++ b/src/GC/GC.jl @@ -8,7 +8,6 @@ See [`gc`](@ref). module GC using ..C: C -using ..Region import ..PythonCall: @pyregion if Base.VERSION ≥ v"1.11" @@ -93,7 +92,7 @@ function enqueue(ptr::C.PyPtr) # If C.CTX.is_initialized is false then the Python interpreter hasn't started yet # or has been finalized; either way attempting to free will cause an error. if ptr != C.PyNULL && C.CTX.is_initialized - if Region.has_tstate() + if C.has_tstate() # An attached state lets us immediately free without blocking a finalizer. C.Py_DecRef(ptr) # We may as well also free any other enqueued objects. @@ -114,7 +113,7 @@ end function enqueue_all(ptrs) if any(!=(C.PyNULL), ptrs) && C.CTX.is_initialized - if Region.has_tstate() + if C.has_tstate() for ptr in ptrs if ptr != C.PyNULL C.Py_DecRef(ptr) @@ -149,7 +148,7 @@ end function _gchook_finalizer(x) if C.CTX.is_initialized finalizer(_gchook_finalizer, x) - if !isempty(QUEUE.items) && Region.has_tstate() + if !isempty(QUEUE.items) && C.has_tstate() unsafe_free_queue() end end diff --git a/src/GIL/GIL.jl b/src/GIL/GIL.jl deleted file mode 100644 index 115e1135..00000000 --- a/src/GIL/GIL.jl +++ /dev/null @@ -1,118 +0,0 @@ -""" - module PythonCall.GIL - -Compatibility API for Python interaction regions. - -See [`lock`](@ref), [`@lock`](@ref), [`unlock`](@ref) and [`@unlock`](@ref). - -!!! warning - - Multi-threading support is experimental and can change without notice. -""" -module GIL - -using ..Region - -if Base.VERSION ≥ v"1.11" - eval( - Expr( - :public, - :lock, - Symbol("@lock"), - :unlock, - Symbol("@unlock"), - ), - ) -end - - -""" - lock(f) - -Compute `f()` in a Python-heavy region and return its result. - -PythonCall APIs already establish the required Python thread state automatically. This -compatibility function can amortize transitions across several operations. - -See [`@lock`](@ref) for the macro form. - -!!! warning - - This function is experimental. Its semantics may be changed without notice. -""" -function lock(f) - token = Region.enter_region() - try - f() - finally - Region.exit_region(token) - end -end - -""" - @lock expr - -Compute `expr` in a Python-heavy region and return its result. - -The macro equivalent of [`lock`](@ref). - -!!! warning - - This macro is experimental. Its semantics may be changed without notice. -""" -macro lock(expr) - quote - token = $Region.enter_region() - try - $(esc(expr)) - finally - $Region.exit_region(token) - end - end -end - -""" - unlock(f) - -Temporarily relinquish Python-related resources, compute `f()`, restore them, and return -the result. Prefer [`PythonCall.@pyregionbreak`](@ref) in new code. - -See [`@unlock`](@ref) for the macro form. - -!!! warning - - This function is experimental. Its semantics may be changed without notice. -""" -function unlock(f) - token = Region.enter_break() - try - f() - finally - Region.exit_break(token) - end -end - -""" - @unlock expr - -Temporarily relinquish Python-related resources, compute `expr`, restore them, and return -the result. Prefer [`PythonCall.@pyregionbreak`](@ref) in new code. - -The macro equivalent of [`unlock`](@ref). - -!!! warning - - This macro is experimental. Its semantics may be changed without notice. -""" -macro unlock(expr) - quote - token = $Region.enter_break() - try - $(esc(expr)) - finally - $Region.exit_break(token) - end - end -end - -end diff --git a/src/JlWrap/C.jl b/src/JlWrap/C.jl index 4f08dab8..8b30d1c3 100644 --- a/src/JlWrap/C.jl +++ b/src/JlWrap/C.jl @@ -448,7 +448,7 @@ function init_c() end function __init__() - init_c() + C.@pyregion init_c() end PyJuliaValue_Check(o) = diff --git a/src/JlWrap/JlWrap.jl b/src/JlWrap/JlWrap.jl index 2ace6b85..dfdf9ff2 100644 --- a/src/JlWrap/JlWrap.jl +++ b/src/JlWrap/JlWrap.jl @@ -12,7 +12,6 @@ using ..C using ..Core using ..Convert using ..GC: GC -using ..GIL import ..PythonCall: pyfunc, diff --git a/src/JlWrap/array.jl b/src/JlWrap/array.jl index dc39da39..837b796a 100644 --- a/src/JlWrap/array.jl +++ b/src/JlWrap/array.jl @@ -3,16 +3,22 @@ const pyjlarraytype = pynew() function pyjl_getaxisindex(x::AbstractUnitRange{<:Integer}, k::Py) if pyisslice(k) a = @pyconvert Union{Int,Nothing} k.start begin - @pyregion errset(pybuiltins.TypeError, "slice components must be integers") - @pyregion pythrow() + @pyregion begin + errset(pybuiltins.TypeError, "slice components must be integers") + pythrow() + end end b = @pyconvert Union{Int,Nothing} k.step begin - @pyregion errset(pybuiltins.TypeError, "slice components must be integers") - @pyregion pythrow() + @pyregion begin + errset(pybuiltins.TypeError, "slice components must be integers") + pythrow() + end end c = @pyconvert Union{Int,Nothing} k.stop begin - @pyregion errset(pybuiltins.TypeError, "slice components must be integers") - @pyregion pythrow() + @pyregion begin + errset(pybuiltins.TypeError, "slice components must be integers") + pythrow() + end end # step defaults to 1 b′ = b === nothing ? 1 : b @@ -26,8 +32,10 @@ function pyjl_getaxisindex(x::AbstractUnitRange{<:Integer}, k::Py) a′ = Int(last(x)) c′ = Int(first(x)) else - @pyregion errset(pybuiltins.ValueError, "step must be non-zero") - @pyregion pythrow() + @pyregion begin + errset(pybuiltins.ValueError, "step must be non-zero") + pythrow() + end end else # start defaults @@ -41,23 +49,29 @@ function pyjl_getaxisindex(x::AbstractUnitRange{<:Integer}, k::Py) if checkbounds(Bool, x, r) return r else - @pyregion errset(pybuiltins.IndexError, "array index out of bounds") - @pyregion pythrow() + @pyregion begin + errset(pybuiltins.IndexError, "array index out of bounds") + pythrow() + end end else j = @pyconvert Int k begin - @pyregion errset( - pybuiltins.TypeError, - "index must be slice or integer, got '$(pytype(k).__name__)'", - ) - @pyregion pythrow() + @pyregion begin + errset( + pybuiltins.TypeError, + "index must be slice or integer, got '$(pytype(k).__name__)'", + ) + pythrow() + end end r = Int(j < 0 ? (last(x) + j + 1) : (first(x) + j)) if checkbounds(Bool, x, r) return r else - @pyregion errset(pybuiltins.IndexError, "array index out of bounds") - @pyregion pythrow() + @pyregion begin + errset(pybuiltins.IndexError, "array index out of bounds") + pythrow() + end end end end @@ -72,8 +86,10 @@ function pyjl_getarrayindices(x::AbstractArray{T,N}, ks::Py) where {T,N} return ans end else - @pyregion errset(pybuiltins.TypeError, "expecting $N indices, got $(pylen(ks))") - @pyregion pythrow() + @pyregion begin + errset(pybuiltins.TypeError, "expecting $N indices, got $(pylen(ks))") + pythrow() + end end elseif N == 1 return (pyjl_getaxisindex(axes(x, 1), ks),) @@ -111,8 +127,10 @@ function pyjlarray_delitem(x::AbstractArray{T,N}, k_::Py) where {T,N} unsafe_pydel(k_) deleteat!(x, k...) else - @pyregion errset(pybuiltins.TypeError, "can only delete from 1D arrays") - @pyregion pythrow() + @pyregion begin + errset(pybuiltins.TypeError, "can only delete from 1D arrays") + pythrow() + end end return Py(nothing) end diff --git a/src/JlWrap/base.jl b/src/JlWrap/base.jl index 29c8ec36..d7094136 100644 --- a/src/JlWrap/base.jl +++ b/src/JlWrap/base.jl @@ -34,59 +34,57 @@ pyconvert_rule_jlvalue(::Type{T}, x::Py) where {T} = function Cjl._pyjl_callmethod(f, self_::C.PyPtr, args_::C.PyPtr, nargs::C.Py_ssize_t) @nospecialize f - @pyregion begin - in_f = false - self = Cjl.PyJuliaValue_GetValue(self_) - try - if nargs == 1 - in_f = true - ans = @pyregionbreak(f(self))::Py - in_f = false - elseif nargs == 2 - arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) - in_f = true - ans = @pyregionbreak(f(self, arg1))::Py - in_f = false - elseif nargs == 3 - arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) - arg2 = pynew(incref(C.PyTuple_GetItem(args_, 2))) - in_f = true - ans = @pyregionbreak(f(self, arg1, arg2))::Py - in_f = false - elseif nargs == 4 - arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) - arg2 = pynew(incref(C.PyTuple_GetItem(args_, 2))) - arg3 = pynew(incref(C.PyTuple_GetItem(args_, 3))) - in_f = true - ans = @pyregionbreak(f(self, arg1, arg2, arg3))::Py - in_f = false - else - @pyregion errset( - pybuiltins.NotImplementedError, - "__jl_callmethod not implemented for this many arguments", - ) - end - return getptr(incref(ans)) - catch exc - if exc isa PyException - Base.GC.@preserve exc C.PyErr_Restore( - incref(exc._t), - incref(exc._v), - incref(exc._b), - ) - return C.PyNULL - else - try - if in_f - return pyjl_handle_error(f, self, exc) - else - @pyregion errset(pyJuliaError, pytuple((pyjl(exc), pyjl(catch_backtrace())))) - return C.PyNULL - end - catch - @pyregion errset(pyJuliaError, "an error occurred while setting an error") + in_f = false + self = Cjl.PyJuliaValue_GetValue(self_) + try + if nargs == 1 + in_f = true + ans = @pyregionbreak(f(self))::Py + in_f = false + elseif nargs == 2 + arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) + in_f = true + ans = @pyregionbreak(f(self, arg1))::Py + in_f = false + elseif nargs == 3 + arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) + arg2 = pynew(incref(C.PyTuple_GetItem(args_, 2))) + in_f = true + ans = @pyregionbreak(f(self, arg1, arg2))::Py + in_f = false + elseif nargs == 4 + arg1 = pynew(incref(C.PyTuple_GetItem(args_, 1))) + arg2 = pynew(incref(C.PyTuple_GetItem(args_, 2))) + arg3 = pynew(incref(C.PyTuple_GetItem(args_, 3))) + in_f = true + ans = @pyregionbreak(f(self, arg1, arg2, arg3))::Py + in_f = false + else + @pyregion errset( + pybuiltins.NotImplementedError, + "__jl_callmethod not implemented for this many arguments", + ) + end + return getptr(incref(ans)) + catch exc + if exc isa PyException + Base.GC.@preserve exc C.PyErr_Restore( + incref(exc._t), + incref(exc._v), + incref(exc._b), + ) + return C.PyNULL + else + try + if in_f + return pyjl_handle_error(f, self, exc) + else + @pyregion errset(pyJuliaError, pytuple((pyjl(exc), pyjl(catch_backtrace())))) return C.PyNULL end + catch + @pyregion errset(pyJuliaError, "an error occurred while setting an error") + return C.PyNULL end end end diff --git a/src/PyMacro/PyMacro.jl b/src/PyMacro/PyMacro.jl index a185ea8e..d8dc45ab 100644 --- a/src/PyMacro/PyMacro.jl +++ b/src/PyMacro/PyMacro.jl @@ -560,7 +560,7 @@ function py_macro_lower(st, body, ans, ex; flavour = :expr) # @jl x elseif @capture(ex, @jl ax_) y = py_macro_lower_jl(st, ax) - py_macro_assign(body, ans, :(@pyregionbreak $y)) + py_macro_assign(body, ans, y) return false # @compile code mode=mode ... @@ -918,21 +918,7 @@ Py({'y': 2}) This macro is experimental. It may be modified or removed in a future release. """ macro py(ex) - lowered = py_macro(ex, __module__, __source__) - assigned = Symbol[] - function find_assigned(x) - x isa Expr || return - if x.head === :(=) && x.args[1] isa Symbol - push!(assigned, x.args[1]) - end - foreach(find_assigned, x.args) - end - find_assigned(lowered) - locals = unique(assigned) - esc(quote - $(isempty(locals) ? nothing : Expr(:local, locals...)) - @pyregion $lowered - end) + esc(py_macro(ex, __module__, __source__)) end end diff --git a/src/PythonCall.jl b/src/PythonCall.jl index 48da84ba..a6dec734 100644 --- a/src/PythonCall.jl +++ b/src/PythonCall.jl @@ -6,8 +6,6 @@ include("API/API.jl") include("Utils/Utils.jl") include("NumpyDates/NumpyDates.jl") include("C/C.jl") -include("Region/Region.jl") -include("GIL/GIL.jl") include("GC/GC.jl") include("Core/Core.jl") include("Convert/Convert.jl") @@ -16,10 +14,6 @@ include("Wrap/Wrap.jl") include("JlWrap/JlWrap.jl") include("Compat/Compat.jl") -function __init__() - Region.start_runtime() -end - # not API but used in tests for k in [ :pyjlanytype, diff --git a/src/Utils/Utils.jl b/src/Utils/Utils.jl index be5d2a5a..4d6ac0e7 100644 --- a/src/Utils/Utils.jl +++ b/src/Utils/Utils.jl @@ -2,40 +2,43 @@ module Utils using Preferences: @load_preference -# Minimal package-local equivalents of Base.OncePerThread and Base.OncePerTask. The Base -# implementations are only available on newer Julia releases, while PythonCall supports -# Julia 1.10. These intentionally provide only the callable interface needed internally. -mutable struct OncePerThread{T,F} - initializer::F - values::Dict{Int,T} - lock::ReentrantLock -end +@static if isdefined(Base, :OncePerThread) + const OncePerThread = Base.OncePerThread + const OncePerTask = Base.OncePerTask +else + # Minimal package-local equivalents for Julia versions where Base does not provide them. + mutable struct OncePerThread{T,F} + initializer::F + values::Dict{Int,T} + lock::ReentrantLock + end -OncePerThread{T}(initializer::F) where {T,F} = - OncePerThread{T,F}(initializer, Dict{Int,T}(), ReentrantLock()) + OncePerThread{T}(initializer::F) where {T,F} = + OncePerThread{T,F}(initializer, Dict{Int,T}(), ReentrantLock()) -function (once::OncePerThread{T})() where {T} - tid = Threads.threadid() - lock(once.lock) - try - return get!(once.values, tid) do - once.initializer()::T + function (once::OncePerThread{T})() where {T} + tid = Threads.threadid() + lock(once.lock) + try + return get!(once.values, tid) do + once.initializer()::T + end + finally + unlock(once.lock) end - finally - unlock(once.lock) end -end -mutable struct OncePerTask{T,F} - initializer::F -end + mutable struct OncePerTask{T,F} + initializer::F + end -OncePerTask{T}(initializer::F) where {T,F} = OncePerTask{T,F}(initializer) + OncePerTask{T}(initializer::F) where {T,F} = OncePerTask{T,F}(initializer) -function (once::OncePerTask{T})() where {T} - get!(task_local_storage(), once) do - once.initializer()::T - end::T + function (once::OncePerTask{T})() where {T} + get!(task_local_storage(), once) do + once.initializer()::T + end::T + end end function getpref(::Type{T}, prefname, envname, default = nothing) where {T} diff --git a/test/Core.jl b/test/Core.jl index e0f9195a..053fbd81 100644 --- a/test/Core.jl +++ b/test/Core.jl @@ -891,11 +891,11 @@ end @testset "gil (#751)" begin o = pyint(751) t = Base.Threads.@spawn begin - PythonCall.GIL.@lock begin + @pyregion begin propertynames(o) end end - PythonCall.GIL.@unlock wait(t) + @pyregionbreak wait(t) end end diff --git a/test/GC.jl b/test/GC.jl index f305c6f7..6b014390 100644 --- a/test/GC.jl +++ b/test/GC.jl @@ -1,7 +1,7 @@ @testitem "GC.gc()" begin let pyobjs = map(pylist, 1:100) - PythonCall.GIL.@unlock begin + @pyregionbreak begin Threads.@threads for obj in pyobjs finalize(obj) end @@ -18,7 +18,7 @@ end @testitem "GC.GCHook" begin let pyobjs = map(pylist, 1:100) - PythonCall.GIL.@unlock begin + @pyregionbreak begin Threads.@threads for obj in pyobjs finalize(obj) end diff --git a/test/GIL.jl b/test/GIL.jl index 57e1fca5..38f01fc8 100644 --- a/test/GIL.jl +++ b/test/GIL.jl @@ -1,10 +1,10 @@ -@testitem "unlock and lock" begin +@testitem "region break and region" begin # This calls Python's time.sleep(1) twice concurrently. Since sleep() unlocks the # GIL, these can happen in parallel if Julia has at least 2 threads. function threaded_sleep() - PythonCall.GIL.unlock() do + @pyregionbreak begin Threads.@threads :static for i = 1:2 - PythonCall.GIL.lock() do + @pyregion begin pyimport("time").sleep(1) end end @@ -20,12 +20,12 @@ end end -@testitem "@unlock and @lock" begin +@testitem "@pyregionbreak and @pyregion" begin # This calls Python's time.sleep(1) twice concurrently. Since sleep() unlocks the # GIL, these can happen in parallel if Julia has at least 2 threads. function threaded_sleep() - PythonCall.GIL.@unlock Threads.@threads :static for i = 1:2 - PythonCall.GIL.@lock pyimport("time").sleep(1) + @pyregionbreak Threads.@threads :static for i = 1:2 + @pyregion pyimport("time").sleep(1) end end # one run to ensure it's compiled From 6625707ee1ae0c198e135846590720fb1d0d337e Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Sat, 3 Oct 2026 18:06:21 +0100 Subject: [PATCH 5/7] Document automatic thread-state migration --- AGENTS.md | 2 ++ CHANGELOG.md | 12 +++++++++ docs/src/faq.md | 4 ++- docs/src/juliacall-reference.md | 3 ++- docs/src/juliacall.md | 23 ---------------- docs/src/pythoncall.md | 3 ++- docs/src/v1-migration-guide.md | 47 +++++++++++++++++++++++++++++++++ 7 files changed, 68 insertions(+), 26 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ce85f833..d5e48b99 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,6 +6,8 @@ - Copy `pysrc/juliacall/juliapkg-dev.json` to `pysrc/juliacall/juliapkg.json` before running (do **not** commit this copy). - Execute with `uv run pytest -s --nbval ./pytest` (add `--cov=pysrc` when coverage is needed). - Sometimes `juliapkg` requires Julia 1.10–1.11; `juliaup` already provides 1.11.7 in this environment. +- `julia --project=docs docs/make.jl` requires a valid Git `origin` so Documenter can infer + source links; it fails during `makedocs` in checkouts without one. The majority of tests live in the Julia package; Python tests cover functionality that cannot be exercised from Julia (e.g., JuliaCall-specific behavior). Run both suites—typically Julia first—in whichever order makes sense. diff --git a/CHANGELOG.md b/CHANGELOG.md index db5ef817..b6dfe2f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,18 @@ * Python errors no longer automatically set `sys.last_traceback` etc. when displayed from Julia. * Added [`fix_qt_plugin_path` preference](@ref pythoncall-config), replacing `CONFIG.auto_fix_qt_plugin_path`. * Removed `PythonCall.CONFIG`. +* Changes to multi-threading: + * PythonCall APIs now manage CPython thread states automatically and can be called from + any Julia task or thread without manually locking the GIL. + * Added the optional, nestable `@pyregion` and `@pyregionbreak` macros. These are + performance and concurrency hints rather than correctness requirements. + * Removed the `PythonCall.GIL` module. Ordinary PythonCall operations replace + `GIL.lock`, and `@pyregionbreak` replaces `GIL.unlock` when an enclosing region needs + to relinquish Python resources around yielding or blocking Julia code. + * JuliaCall now relinquishes and restores Python resources automatically while running + Julia code, including Python-to-Julia callbacks. + * Python-object finalizers no longer attach a Python thread state. Pending decrefs are + queued until a state is already attached or `PythonCall.GC.gc()` drains the queue. * Changes to `PythonCall.GC` (now more like `Base.GC`): * `enable(true)` replaces `enable()`. * `enable(false)` replaces `disable()`. diff --git a/docs/src/faq.md b/docs/src/faq.md index 260f980f..69df5394 100644 --- a/docs/src/faq.md +++ b/docs/src/faq.md @@ -12,7 +12,9 @@ Alternatively, to force PyCall to use the same interpreter as PythonCall, set th ## [Is PythonCall/JuliaCall thread safe?](@id faq-multi-threading) -Yes, as of v0.9.22, provided you handle the GIL correctly. See the guides for +Yes. PythonCall and JuliaCall manage the required Python thread states automatically, so +PythonCall operations can be used from any Julia task or thread without explicit GIL +handling. See the guides for [PythonCall](@ref jl-multi-threading) and [JuliaCall](@ref py-multi-threading). Before, tricks such as disabling the garbage collector were required. See the diff --git a/docs/src/juliacall-reference.md b/docs/src/juliacall-reference.md index 51fda18e..844fe0dd 100644 --- a/docs/src/juliacall-reference.md +++ b/docs/src/juliacall-reference.md @@ -70,7 +70,8 @@ and ending with `__` (since these are Python special methods) or starting with ` - `jl_callback(*args, **kwargs)`: Calls the Julia object with the given arguments. Unlike ordinary calling syntax, the arguments are passed as `Py` objects instead of being converted. -- `jl_call_nogil(*args, **kwargs)`: Call this with the GIL disabled. +- `jl_call_nogil(*args, **kwargs)`: Compatibility spelling for an ordinary call. Python + resources are relinquished automatically while Julia code runs. - `jl_display()`: Display the object using Julia's display mechanism. - `jl_eval(expr)`: If the object is a Julia `Module`, evaluates the given expression. - `jl_help()`: Display help for the object. diff --git a/docs/src/juliacall.md b/docs/src/juliacall.md index 724c3c4b..bdac1b87 100644 --- a/docs/src/juliacall.md +++ b/docs/src/juliacall.md @@ -171,29 +171,6 @@ PythonCall operations nested inside Julia callbacks are safe without explicit re You can also use [multi-threading from Julia](@ref jl-multi-threading). -### Caveat: Julia's task scheduler - -If you try the above example with a Julia function that yields to the task scheduler, -such as `sleep` instead of `Libc.systemsleep`, then you will likely experience a hang. - -In this case, you need to yield back to Julia's scheduler periodically to allow the task -to continue. You can use the following pattern instead of `wait(fs)`: -```python -jl_yield = getattr(jl, "yield") -while True: - # yield to Julia's task scheduler - jl_yield() - # wait for up to 0.1 seconds for the threads to finish - state = wait(fs, timeout=0.1) - # if they finished then stop otherwise try again - if not state.not_done: - break -``` - -Set the `timeout` parameter smaller to let Julia's scheduler cycle more frequently. - -Future versions of JuliaCall may provide tooling to make this simpler. - ### [Caveat: Signal handling](@id py-multi-threading-signal-handling) We recommend setting [`PYTHON_JULIACALL_HANDLE_SIGNALS=yes`](@ref julia-config) diff --git a/docs/src/pythoncall.md b/docs/src/pythoncall.md index 3659329f..8d45cfd7 100644 --- a/docs/src/pythoncall.md +++ b/docs/src/pythoncall.md @@ -502,4 +502,5 @@ If Julia's GC collects Python objects while no Python thread state is already at those objects are queued rather than making a finalizer block while attaching a state. If you find you have many Python objects not being deleted, you can call -[`PythonCall.GC.gc()`](@ref) or `GC.gc()` to clear the queue. +[`PythonCall.GC.gc()`](@ref) to clear the queue. An ordinary `GC.gc()` may also clear it +when a Python thread state is already attached, such as inside `@pyregion`. diff --git a/docs/src/v1-migration-guide.md b/docs/src/v1-migration-guide.md index 3300df85..f08a71c4 100644 --- a/docs/src/v1-migration-guide.md +++ b/docs/src/v1-migration-guide.md @@ -77,6 +77,53 @@ The `PythonCall.CONFIG.auto_fix_qt_plugin_path` config has been replaced with th `pkg> preference add PythonCall fix_qt_plugin_path=false` or the env var `JULIA_PYTHONCALL_FIX_QT_PLUGIN_PATH=0`. +## Multi-threading + +PythonCall now manages CPython thread states automatically. PythonCall operations can be +used from any Julia task or thread, including Julia code entered through JuliaCall, without +manually locking or unlocking the GIL. + +The `PythonCall.GIL` module has been removed: + +* Remove `PythonCall.GIL.lock` and `PythonCall.GIL.@lock` around individual PythonCall + operations; each operation establishes the Python state it needs automatically. +* For a sequence of straight-line, Python-heavy operations, `@pyregion` is an optional + optimization which amortizes thread-state transitions. +* Instead of `PythonCall.GIL.unlock(f)` or `PythonCall.GIL.@unlock expr` inside such a + region, use `@pyregionbreak f()` or `@pyregionbreak expr`. A region break is useful + around Julia code which may yield, wait, or block cooperatively. + +For example, code which previously managed the GIL explicitly: + +```julia +PythonCall.GIL.@lock begin + x = pyimport("example").make_value() + PythonCall.GIL.@unlock wait(event) + x.finish() +end +``` + +can be written as: + +```julia +@pyregion begin + x = pyimport("example").make_value() + @pyregionbreak wait(event) + x.finish() +end +``` + +The outer `@pyregion` can also be omitted; the PythonCall operations remain correct, and +`wait(event)` then needs no annotation. Both macros nest safely. + +JuliaCall similarly relinquishes Python resources automatically while arbitrary Julia code +runs. Ordinary calls should be used instead of the old `_jl_call_nogil` spelling. The v1 +`jl_call_nogil` method remains available for compatibility but behaves like an ordinary +call. + +Finalizers no longer attach a Python thread state and block waiting for Python. If prompt +cleanup matters, call `PythonCall.GC.gc()` at a suitable point to drain queued decrefs. + ## `PythonCall.GC` This submodule has been changed to closer mimic the `Base.GC` API. From f82658e7dd95ec16b7a715e2aa59f68822cc89da Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Sat, 3 Oct 2026 18:55:59 +0100 Subject: [PATCH 6/7] Remove jl_call_nogil and benchmark regions --- CHANGELOG.md | 1 + benchmark/benchmarks.jl | 43 +++++++++++++++++++++++++++++++++ docs/src/juliacall-reference.md | 2 -- docs/src/juliacall.md | 18 ++++++++++++++ docs/src/v1-migration-guide.md | 5 ++-- pytest/test_all.py | 8 +++--- src/JlWrap/any.jl | 22 ----------------- 7 files changed, 68 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6dfe2f3..7c758436 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ to relinquish Python resources around yielding or blocking Julia code. * JuliaCall now relinquishes and restores Python resources automatically while running Julia code, including Python-to-Julia callbacks. + * Removed JuliaCall's `jl_call_nogil` method; ordinary calls now provide its behavior. * Python-object finalizers no longer attach a Python thread state. Pending decrefs are queued until a state is already attached or `PythonCall.GC.gc()` drains the queue. * Changes to `PythonCall.GC` (now more like `Base.GC`): diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 1958f89a..7854caf0 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -50,6 +50,49 @@ end SUITE["basic"]["@py"]["pydict"]["init"] = @benchmarkable test_atpy(Val(false)) SUITE["basic"]["@py"]["pydict"]["pydel"] = @benchmarkable test_atpy(Val(true)) +function test_region_pythoncall(x, n) + ans = 0 + for _ = 1:n + ans += pylen(x) + end + return ans +end + +function test_region_pythoncall_outer(x, n) + @pyregion begin + ans = 0 + for _ = 1:n + ans += pylen(x) + end + return ans + end +end + +function test_region_capi(x, n) + @pyregion begin + ans = 0 + for _ = 1:n + ans += PythonCall.C.PyObject_Length(x) + end + return ans + end +end + +const REGION_BENCHMARK_LENGTH = 1000 + +SUITE["region"]["pythoncall"] = @benchmarkable( + test_region_pythoncall(x, REGION_BENCHMARK_LENGTH), + setup=(x = pytuple((1, 2, 3))), +) +SUITE["region"]["pythoncall_outer"] = @benchmarkable( + test_region_pythoncall_outer(x, REGION_BENCHMARK_LENGTH), + setup=(x = pytuple((1, 2, 3))), +) +SUITE["region"]["capi_outer"] = @benchmarkable( + test_region_capi(x, REGION_BENCHMARK_LENGTH), + setup=(x = pytuple((1, 2, 3))), +) + include("gcbench.jl") using .GCBench: append_lots diff --git a/docs/src/juliacall-reference.md b/docs/src/juliacall-reference.md index 844fe0dd..43496c96 100644 --- a/docs/src/juliacall-reference.md +++ b/docs/src/juliacall-reference.md @@ -70,8 +70,6 @@ and ending with `__` (since these are Python special methods) or starting with ` - `jl_callback(*args, **kwargs)`: Calls the Julia object with the given arguments. Unlike ordinary calling syntax, the arguments are passed as `Py` objects instead of being converted. -- `jl_call_nogil(*args, **kwargs)`: Compatibility spelling for an ordinary call. Python - resources are relinquished automatically while Julia code runs. - `jl_display()`: Display the object using Julia's display mechanism. - `jl_eval(expr)`: If the object is a Julia `Module`, evaluates the given expression. - `jl_help()`: Display help for the object. diff --git a/docs/src/juliacall.md b/docs/src/juliacall.md index bdac1b87..b41f3733 100644 --- a/docs/src/juliacall.md +++ b/docs/src/juliacall.md @@ -171,6 +171,24 @@ PythonCall operations nested inside Julia callbacks are safe without explicit re You can also use [multi-threading from Julia](@ref jl-multi-threading). +### Caveat: Julia's task scheduler + +If you use a Julia function which yields to the task scheduler, such as `sleep` instead of +`Libc.systemsleep` in the example above, the Python thread waiting for the futures must +periodically yield to Julia so that the Julia tasks can finish: + +```python +jl_yield = getattr(jl, "yield") +while True: + jl_yield() + state = wait(fs, timeout=0.1) + if not state.not_done: + break +``` + +This scheduler requirement is independent of Python thread-state management; ordinary +JuliaCall calls still relinquish and restore Python resources automatically. + ### [Caveat: Signal handling](@id py-multi-threading-signal-handling) We recommend setting [`PYTHON_JULIACALL_HANDLE_SIGNALS=yes`](@ref julia-config) diff --git a/docs/src/v1-migration-guide.md b/docs/src/v1-migration-guide.md index f08a71c4..25357887 100644 --- a/docs/src/v1-migration-guide.md +++ b/docs/src/v1-migration-guide.md @@ -117,9 +117,8 @@ The outer `@pyregion` can also be omitted; the PythonCall operations remain corr `wait(event)` then needs no annotation. Both macros nest safely. JuliaCall similarly relinquishes Python resources automatically while arbitrary Julia code -runs. Ordinary calls should be used instead of the old `_jl_call_nogil` spelling. The v1 -`jl_call_nogil` method remains available for compatibility but behaves like an ordinary -call. +runs. The `_jl_call_nogil` and `jl_call_nogil` methods have been removed; use an ordinary +call instead. Finalizers no longer attach a Python thread state and block waiting for Python. If prompt cleanup matters, call `PythonCall.GC.gc()` at a suitable point to drain queued decrefs. diff --git a/pytest/test_all.py b/pytest/test_all.py index 0267f7df..d23094f4 100644 --- a/pytest/test_all.py +++ b/pytest/test_all.py @@ -113,20 +113,20 @@ def test_julia_gc(): @pytest.mark.parametrize("yld", [True, False]) -def test_call_nogil(yld): - """Tests that we can execute Julia code in parallel by releasing the GIL.""" +def test_parallel_call(yld): + """Tests that ordinary calls execute Julia code in parallel.""" from concurrent.futures import ThreadPoolExecutor, wait from time import time from juliacall import Main as jl - # julia implementation of sleep which unlocks the GIL + # Julia implementations of sleep which do and do not yield. if yld: # use sleep, which yields jsleep = jl.sleep else: # use Libc.systemsleep which does not yield jsleep = jl.Libc.systemsleep - jsleep = jsleep.jl_call_nogil + assert not hasattr(jsleep, "jl_call_nogil") jyield = getattr(jl, "yield") # precompile jsleep(0.01) diff --git a/src/JlWrap/any.jl b/src/JlWrap/any.jl index d65cb597..d1941387 100644 --- a/src/JlWrap/any.jl +++ b/src/JlWrap/any.jl @@ -90,24 +90,6 @@ end pyjl_handle_error_type(::typeof(pyjlany_callback), self, exc::MethodError) = exc.f === self ? pybuiltins.TypeError : PyNULL -function pyjlany_call_nogil(self, args_::Py, kwargs_::Py) - if pylen(kwargs_) > 0 - args = pyconvert(Vector{Any}, args_) - kwargs = pyconvert(Dict{Symbol,Any}, kwargs_) - ans = pyjl(self(args...; kwargs...)) - elseif pylen(args_) > 0 - args = pyconvert(Vector{Any}, args_) - ans = pyjl(self(args...)) - else - ans = pyjl(self()) - end - unsafe_pydel(args_) - unsafe_pydel(kwargs_) - ans -end -pyjl_handle_error_type(::typeof(pyjlany_call_nogil), self, exc::MethodError) = - exc.f === self ? pybuiltins.TypeError : PyNULL - function pyjlany_getitem(self, k_::Py) if self isa Type if pyistuple(k_) @@ -607,10 +589,6 @@ class Jl(JlBase2): return self._jl_callmethod($(pyjl_methodnum(Py))) def jl_callback(self, *args, **kwargs): return self._jl_callmethod($(pyjl_methodnum(pyjlany_callback)), args, kwargs) - def jl_call_nogil(self, *args, **kwargs): - '''Compatibility alias for calling this Julia object. Python resources are - relinquished automatically while Julia code runs.''' - return self._jl_callmethod($(pyjl_methodnum(pyjlany_call_nogil)), args, kwargs) def _repr_mimebundle_(self, include=None, exclude=None): return self._jl_callmethod($(pyjl_methodnum(pyjlany_mimebundle)), include, exclude) @property From fdbbb035e13acfafb646ac4fac73bd176e64a270 Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Sun, 4 Oct 2026 20:50:11 +0100 Subject: [PATCH 7/7] Fast-path already attached Python regions --- src/C/regions.jl | 109 +++++++++++++++++++++++++++++++++++++++++------ test/C.jl | 58 +++++++++++++++++++++++++ test/Region.jl | 42 ------------------ 3 files changed, 154 insertions(+), 55 deletions(-) delete mode 100644 test/Region.jl diff --git a/src/C/regions.jl b/src/C/regions.jl index 26af46fe..81b5747d 100644 --- a/src/C/regions.jl +++ b/src/C/regions.jl @@ -1,27 +1,83 @@ const PyThreadStatePtr = Ptr{Cvoid} -mutable struct ThreadState - sem::Base.Semaphore - tstate::PyThreadStatePtr -end -ThreadState() = ThreadState(Base.Semaphore(1), C_NULL) - mutable struct TaskState + # CPython state used by this task for the duration of its outermost region. tstate::PyThreadStatePtr + # Semaphore protecting `tstate`; `nothing` when the task is outside a session. sem::Union{Nothing,Base.Semaphore} + # Whether `tstate` is currently attached to the task's pinned OS thread. attached::Bool + # Julia thread to which the task is pinned while its session is active. tid::Int + # Stickiness to restore when the task's outermost region or break finishes. oldsticky::Bool end TaskState() = TaskState(C_NULL, nothing, false, 0, false) +mutable struct ThreadState + # Serializes tasks which use the persistent CPython state on this Julia thread. + sem::Base.Semaphore + # PythonCall-owned CPython state for this Julia thread, allocated lazily. + tstate::PyThreadStatePtr + # Task which currently holds `sem` with a CPython state attached, if any. + task::Union{Nothing,Task} + # State belonging to `task`; cached so nested regions need no task-local lookup. + task_state::Union{Nothing,TaskState} +end +ThreadState() = ThreadState(Base.Semaphore(1), C_NULL, nothing, nothing) + const THREAD_STATE = Utils.OncePerThread{ThreadState}(ThreadState) const TASK_STATE = Utils.OncePerTask{TaskState}(TaskState) +# A region session starts at the outermost region entered by a Julia task and ends +# when that region exits. The task is made sticky for the whole session: CPython +# thread states must be restored on the same OS thread on which they were saved. +# The corresponding ThreadState semaphore gives the task exclusive use of that +# thread's persistent PyThreadState. A Python-originated call already has a CPython +# state attached; that state is borrowed instead, but is protected by the same +# semaphore so PythonCall's accounting remains identical. +# +# Nested regions merely return `:noop`. A break saves/detaches the CPython state, +# clears the thread cache, and releases the semaphore, allowing other sticky tasks +# to use this Julia thread while the original task yields. On return it reacquires +# the semaphore before restoring both the CPython state and cache. Consequently the +# cache is populated exactly while a task owns the semaphore with Python attached. +# This invariant is what makes its lock-free fast path safe. +# +# An attached CPython state is also a still cheaper fast path for @pyregion itself. +# Given that PythonCall is the only Julia-side entry to Python, an attached state +# means either that an enclosing region already did the bookkeeping or that Python +# called into Julia and lent us its state. In either case a nested region has no +# transition to perform. Conversely, a region inside a break observes no attached +# state and takes the full path below, reacquiring and restoring its task state. + current_tstate() = PyThreadState_GetUnchecked() has_tstate() = CTX.is_initialized && current_tstate() != C_NULL +# An attached task is pinned to this Julia thread and owns its semaphore, so the +# thread-local cache cannot change underneath it. This makes nested regions avoid +# the considerably more expensive OncePerTask lookup. The cache is cleared before +# releasing the semaphore and restored after reacquiring it around a region break. +function current_task_state(task::Task) + ts = THREAD_STATE() + return ts.task === task ? ts.task_state::TaskState : TASK_STATE() +end + +function set_task!(ts::ThreadState, task::Task, s::TaskState) + ts.task = task + ts.task_state = s + return +end + +function clear_task!(ts::ThreadState) + ts.task = nothing + ts.task_state = nothing + return +end + function reset!(s::TaskState) + # `oldsticky` is deliberately left alone: start_session! overwrites it before + # the TaskState becomes observable as an active session again. s.tstate = C_NULL s.sem = nothing s.attached = false @@ -30,6 +86,8 @@ function reset!(s::TaskState) end function start_session!(task::Task, s::TaskState) + # Pin before recording the thread and looking up its ThreadState. Once sticky, + # cooperative scheduling cannot resume this task on another Julia thread. s.oldsticky = task.sticky task.sticky = true s.tid = Threads.threadid() @@ -45,6 +103,8 @@ end function enter_region(task::Task, s::TaskState) if s.tstate != C_NULL + # An existing session is either already attached (ordinary nesting), or + # temporarily detached because this region is nested inside a break. check_thread(task, s) if s.attached return :noop @@ -53,6 +113,7 @@ function enter_region(task::Task, s::TaskState) try PyEval_RestoreThread(s.tstate) s.attached = true + set_task!(THREAD_STATE(), task, s) catch Base.release(s.sem::Base.Semaphore) rethrow() @@ -73,8 +134,10 @@ function enter_region(task::Task, s::TaskState) acquired = true current = current_tstate() if current != C_NULL + # Calls originating in Python must return with its state still attached. s.tstate = current s.attached = true + set_task!(ts, task, s) return :root_borrowed end if ts.tstate == C_NULL @@ -84,6 +147,7 @@ function enter_region(task::Task, s::TaskState) s.tstate = ts.tstate PyEval_RestoreThread(s.tstate) s.attached = true + set_task!(ts, task, s) return :root_owned catch acquired && Base.release(ts.sem) @@ -94,23 +158,28 @@ function enter_region(task::Task, s::TaskState) end function exit_region(task::Task, s::TaskState, token) + # Tokens encode precisely which work enter_region performed, so every path + # reverses only its own transition and nested :noop regions do no attach work. token === :noop && return check_thread(task, s) if token === :detach saved = PyEval_SaveThread() @assert saved == s.tstate s.attached = false + clear_task!(THREAD_STATE()) Base.release(s.sem::Base.Semaphore) elseif token === :root_owned saved = PyEval_SaveThread() @assert saved == s.tstate sem, oldsticky = s.sem::Base.Semaphore, s.oldsticky + clear_task!(THREAD_STATE()) reset!(s) Base.release(sem) task.sticky = oldsticky elseif token === :root_borrowed @assert current_tstate() == s.tstate sem, oldsticky = s.sem::Base.Semaphore, s.oldsticky + clear_task!(THREAD_STATE()) reset!(s) Base.release(sem) task.sticky = oldsticky @@ -122,14 +191,18 @@ end function enter_break(task::Task, s::TaskState) if s.tstate != C_NULL + # Only the outermost break detaches. Further breaks while detached are no-ops. check_thread(task, s) !s.attached && return :noop saved = PyEval_SaveThread() @assert saved == s.tstate s.attached = false + clear_task!(THREAD_STATE()) Base.release(s.sem::Base.Semaphore) return :restore end + # Outside a region, a break matters only in a Python-originated callback. It + # borrows and detaches that state so yielding Julia code cannot retain Python. current_tstate() == C_NULL && return :noop ts = try @@ -170,8 +243,12 @@ function exit_break(task::Task, s::TaskState, token) Base.acquire(s.sem::Base.Semaphore) PyEval_RestoreThread(s.tstate) s.attached = true + set_task!(THREAD_STATE(), task, s) if token === :root_restore + # The borrowed Python state stays attached for the caller, but the temporary + # Julia session and its cache ownership end here. sem, oldsticky = s.sem::Base.Semaphore, s.oldsticky + clear_task!(THREAD_STATE()) reset!(s) Base.release(sem) task.sticky = oldsticky @@ -191,13 +268,19 @@ blocks cooperatively in [`@pyregionbreak`](@ref). """ macro pyregion(ex) quote - local task = current_task() - local state = $TASK_STATE() - local token = $enter_region(task, state) - try + if $current_tstate() != C_NULL + # The CPython TLS lookup is enough to prove that this region is nested + # or Python-originated; avoid both Julia task- and thread-local lookups. $(esc(ex)) - finally - $exit_region(task, state, token) + else + local task = current_task() + local state = $current_task_state(task) + local token = $enter_region(task, state) + try + $(esc(ex)) + finally + $exit_region(task, state, token) + end end end end @@ -212,7 +295,7 @@ PythonCall operations still work automatically, and both kinds of region nest fr macro pyregionbreak(ex) quote local task = current_task() - local state = $TASK_STATE() + local state = $current_task_state(task) local token = $enter_break(task, state) try $(esc(ex)) diff --git a/test/C.jl b/test/C.jl index 69f8697a..c1ffdded 100644 --- a/test/C.jl +++ b/test/C.jl @@ -16,3 +16,61 @@ @test PythonCall.python_version().major == 3 end end + +@testitem "Python regions and task-safe thread states" setup = [Setup] begin + using Base.Threads + + @test PythonCall.C.PyThreadState_GetUnchecked() == C_NULL + @test @pyregion pyconvert(Int, pyint(12)) == 12 + @test PythonCall.C.PyThreadState_GetUnchecked() == C_NULL + + # Nested regions reuse the state cached by the owning Julia thread. A break + # must relinquish that cache while yielding and restore it on return. + @pyregion begin + local state = PythonCall.C.TASK_STATE() + local thread_state = PythonCall.C.THREAD_STATE() + @test thread_state.task === current_task() + @test thread_state.task_state === state + @pyregion @test PythonCall.C.current_task_state(current_task()) === state + @pyregionbreak begin + @test thread_state.task === nothing + yield() + end + @test thread_state.task === current_task() + @test thread_state.task_state === state + end + @test PythonCall.C.THREAD_STATE().task === nothing + + @test @pyregion begin + @pyregion pyconvert(Int, pyint(1)) == 1 + @pyregionbreak begin + yield() + @pyregion pyconvert(Int, pyint(2)) == 2 + @pyregionbreak yield() + end + pyconvert(Int, pyint(3)) == 3 + end + + @test_throws ErrorException @pyregion @pyregionbreak error("region exception") + @test PythonCall.C.PyThreadState_GetUnchecked() == C_NULL + @test_throws PyException @pyregion pybuiltins.int("not an integer") + @test pyconvert(Int, pyint(5)) == 5 + + results = fetch.([@spawn begin + total = 0 + for i in 1:100 + total += pyconvert(Int, pyint(i)) + @pyregionbreak yield() + end + total + end for _ in 1:max(8, 2nthreads())]) + @test all(==(5050), results) + + f = pyfunc() do + yield() + pyconvert(Int, pybuiltins.sum([1, 2, 3])) + end + @test pyconvert(Int, f()) == 6 + + @test pyconvert(Int, @py 1 + @jl(pyconvert(Int, pyint(2)))) == 3 +end diff --git a/test/Region.jl b/test/Region.jl deleted file mode 100644 index b000ad52..00000000 --- a/test/Region.jl +++ /dev/null @@ -1,42 +0,0 @@ -using TestItemRunner - -@testitem "Python regions and task-safe thread states" setup = [Setup] begin - using Base.Threads - - @test PythonCall.C.PyThreadState_GetUnchecked() == C_NULL - @test @pyregion pyconvert(Int, pyint(12)) == 12 - @test PythonCall.C.PyThreadState_GetUnchecked() == C_NULL - - @test @pyregion begin - @pyregion pyconvert(Int, pyint(1)) == 1 - @pyregionbreak begin - yield() - @pyregion pyconvert(Int, pyint(2)) == 2 - @pyregionbreak yield() - end - pyconvert(Int, pyint(3)) == 3 - end - - @test_throws ErrorException @pyregion @pyregionbreak error("region exception") - @test PythonCall.C.PyThreadState_GetUnchecked() == C_NULL - @test_throws PyException @pyregion pybuiltins.int("not an integer") - @test pyconvert(Int, pyint(5)) == 5 - - results = fetch.([@spawn begin - total = 0 - for i in 1:100 - total += pyconvert(Int, pyint(i)) - @pyregionbreak yield() - end - total - end for _ in 1:max(8, 2nthreads())]) - @test all(==(5050), results) - - f = pyfunc() do - yield() - pyconvert(Int, pybuiltins.sum([1, 2, 3])) - end - @test pyconvert(Int, f()) == 6 - - @test pyconvert(Int, @py 1 + @jl(pyconvert(Int, pyint(2)))) == 3 -end