From 4cf99182a4b236b2dd0504248fa3118ee59939c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= Date: Fri, 11 Sep 2026 05:38:03 -0500 Subject: [PATCH 1/3] Resolve compiled CPU kernels through a fast path Every launch went through `clfunction`, which built a compiler config, looked up the method instance, allocated a `CompilerJob`, queried GPUCompiler's cache and hashed its way into `_kernel_instances` before returning the `HostKernel` it had returned last time. Keep the resolved `HostKernel` per `HostKernel{F, tt}` type together with the world age and context it was resolved in, and return it directly while no method has been defined since and the context is unchanged. Reflection (a set `compile_hook`) and explicit compiler keywords bypass the fast path. Reading the `compile_hook` scoped value allocates, so it is only read when a dynamic scope is active. The `Ref` used to find the kernel for the current context and the `@something` in `compile_or_lookup` are replaced with plain control flow, which also removes a boxed variable captured by the `get!` closure. A warm launch now allocates once, for the event handle. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01VHciC8x39gm97sABrSvBkt --- src/pocl/compiler/execution.jl | 57 ++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 10 deletions(-) diff --git a/src/pocl/compiler/execution.jl b/src/pocl/compiler/execution.jl index 5b360842d..4e128b110 100644 --- a/src/pocl/compiler/execution.jl +++ b/src/pocl/compiler/execution.jl @@ -193,8 +193,37 @@ end const clfunction_lock = ReentrantLock() +# `HostKernel` with the world age and context it was resolved in; valid as long as no method +# has been defined since and the context is unchanged. +struct ResolvedKernel + world::UInt + context::nanoOpenCL.Context + kernel::Any +end + +# `HostKernel{F, tt}` instances keyed by their type +const _kernel_fastpath = Dict{DataType, ResolvedKernel}() + +# On Julia 1.11 to 1.13 reading a `ScopedValue` allocates; outside of any dynamic scope it +# holds its default, so the read is skipped there. +@static if v"1.11" <= VERSION < v"1.14-" + @inline compile_hook_set() = Core.current_scope() !== nothing && GPUCompiler.compile_hook[] !== nothing +else + @inline compile_hook_set() = GPUCompiler.compile_hook[] !== nothing +end + function clfunction(f::F, tt::TT = Tuple{}; kwargs...) where {F, TT} Base.@lock clfunction_lock begin + ctx = context() + world = Base.get_world_counter() + cacheable = isempty(kwargs) && !compile_hook_set() + if cacheable + entry = get(_kernel_fastpath, HostKernel{F, tt}, nothing) + if entry !== nothing && entry.world == world && entry.context === ctx + return entry.kernel::HostKernel{F, tt} + end + end + config = compiler_config(device(); kwargs...)::OpenCLCompilerConfig source = methodinstance(F, tt) job = CompilerJob(source, config) @@ -203,28 +232,34 @@ function clfunction(f::F, tt::TT = Tuple{}; kwargs...) where {F, TT} # Resolve the cl.Kernel for the active context. Linear scan over the # session-local cache; almost always n=1, so this is one `===` compare. - ctx = context() - kernel = Ref{nanoOpenCL.Kernel}() + cached = nothing @inbounds for (cached_ctx, cached_kernel) in res.kernels if cached_ctx === ctx - kernel[] = cached_kernel + cached = cached_kernel break end end - if !isassigned(kernel) - kernel[] = link_kernel(job, res.obj::Vector{UInt8}, res.entry::String) + kernel = if cached === nothing + linked = link_kernel(job, res.obj::Vector{UInt8}, res.entry::String) # Don't cache session-local kernel handles while precompiling: the # results struct is serialized into the package image along with its # CodeInstance, and the handles would come back dangling. if ccall(:jl_generating_output, Cint, ()) != 1 - push!(res.kernels, (ctx, kernel[])) + push!(res.kernels, (ctx, linked)) end + linked + else + cached end - h = hash(kernel[], hash(f, hash(tt))) - return get!(_kernel_instances, h) do - HostKernel{F, tt}(f, kernel[], res.device_rng) + h = hash(kernel, hash(f, hash(tt))) + hostkernel = get!(_kernel_instances, h) do + HostKernel{F, tt}(f, kernel, res.device_rng) end::HostKernel{F, tt} + if cacheable + _kernel_fastpath[HostKernel{F, tt}] = ResolvedKernel(world, ctx, hostkernel) + end + return hostkernel end end @@ -241,7 +276,9 @@ function compile_or_lookup(@nospecialize(job::CompilerJob))::OpenCLResults res = GPUCompiler.cached_results(OpenCLResults, job) if res === nothing || res.obj === nothing || GPUCompiler.compile_hook[] !== nothing compiled = compile_to_obj(job) - res = @something res GPUCompiler.cached_results(OpenCLResults, job) + if res === nothing + res = GPUCompiler.cached_results(OpenCLResults, job)::OpenCLResults + end res.obj = compiled.obj res.entry = compiled.entry res.device_rng = compiled.device_rng From 113ec3db29b980cb689127126fddf7d9e0320cfa Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Mon, 14 Sep 2026 10:53:19 +0200 Subject: [PATCH 2/3] Rely on GPUCompiler for cached CPU kernels Avoid duplicating GPUCompiler cache invalidation in a separate fast path. Instead, let compile_or_lookup specialize on the bounded back-end job type so CompilerJob remains allocation-free, and keep the method out of line to avoid duplicating it per kernel. This also preserves distinct HostKernel instances for same-type callables carrying different state. --- src/pocl/compiler/execution.jl | 41 +++++----------------------------- 1 file changed, 6 insertions(+), 35 deletions(-) diff --git a/src/pocl/compiler/execution.jl b/src/pocl/compiler/execution.jl index 4e128b110..4bee8ba72 100644 --- a/src/pocl/compiler/execution.jl +++ b/src/pocl/compiler/execution.jl @@ -193,37 +193,8 @@ end const clfunction_lock = ReentrantLock() -# `HostKernel` with the world age and context it was resolved in; valid as long as no method -# has been defined since and the context is unchanged. -struct ResolvedKernel - world::UInt - context::nanoOpenCL.Context - kernel::Any -end - -# `HostKernel{F, tt}` instances keyed by their type -const _kernel_fastpath = Dict{DataType, ResolvedKernel}() - -# On Julia 1.11 to 1.13 reading a `ScopedValue` allocates; outside of any dynamic scope it -# holds its default, so the read is skipped there. -@static if v"1.11" <= VERSION < v"1.14-" - @inline compile_hook_set() = Core.current_scope() !== nothing && GPUCompiler.compile_hook[] !== nothing -else - @inline compile_hook_set() = GPUCompiler.compile_hook[] !== nothing -end - function clfunction(f::F, tt::TT = Tuple{}; kwargs...) where {F, TT} Base.@lock clfunction_lock begin - ctx = context() - world = Base.get_world_counter() - cacheable = isempty(kwargs) && !compile_hook_set() - if cacheable - entry = get(_kernel_fastpath, HostKernel{F, tt}, nothing) - if entry !== nothing && entry.world == world && entry.context === ctx - return entry.kernel::HostKernel{F, tt} - end - end - config = compiler_config(device(); kwargs...)::OpenCLCompilerConfig source = methodinstance(F, tt) job = CompilerJob(source, config) @@ -232,6 +203,7 @@ function clfunction(f::F, tt::TT = Tuple{}; kwargs...) where {F, TT} # Resolve the cl.Kernel for the active context. Linear scan over the # session-local cache; almost always n=1, so this is one `===` compare. + ctx = context() cached = nothing @inbounds for (cached_ctx, cached_kernel) in res.kernels if cached_ctx === ctx @@ -253,13 +225,9 @@ function clfunction(f::F, tt::TT = Tuple{}; kwargs...) where {F, TT} end h = hash(kernel, hash(f, hash(tt))) - hostkernel = get!(_kernel_instances, h) do + return get!(_kernel_instances, h) do HostKernel{F, tt}(f, kernel, res.device_rng) end::HostKernel{F, tt} - if cacheable - _kernel_fastpath[HostKernel{F, tt}] = ResolvedKernel(world, ctx, hostkernel) - end - return hostkernel end end @@ -272,7 +240,10 @@ end # Julia's code cache, so the post-compile `cached_results` re-fetch is guaranteed to # succeed. The `compile_hook` check additionally forces the compile path so # reflection-style consumers (`@device_code_*`) observe the compilation even on a hit. -function compile_or_lookup(@nospecialize(job::CompilerJob))::OpenCLResults +# Keep this specialized so the caller can avoid boxing `CompilerJob`. Its type parameters +# only identify the target and compiler parameters, so this is bounded per back-end rather +# than specialized for every kernel; `@noinline` keeps the body out of each `clfunction`. +@noinline function compile_or_lookup(job::CompilerJob)::OpenCLResults res = GPUCompiler.cached_results(OpenCLResults, job) if res === nothing || res.obj === nothing || GPUCompiler.compile_hook[] !== nothing compiled = compile_to_obj(job) From 0bafbf5b9f59b9bb93fdfbfb2ed118c4b154722c Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Mon, 14 Sep 2026 11:06:45 +0200 Subject: [PATCH 3/3] Fix integer argument forwarding in POCL set_args! Use a separate recursive helper so a leading Int kernel argument is not interpreted as the argument index. --- src/pocl/nanoOpenCL.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pocl/nanoOpenCL.jl b/src/pocl/nanoOpenCL.jl index e3adadbf1..34450bbdc 100644 --- a/src/pocl/nanoOpenCL.jl +++ b/src/pocl/nanoOpenCL.jl @@ -1293,11 +1293,11 @@ function set_arg!(k::Kernel, idx::Integer, arg::T) where {T} return k end -set_args!(k::Kernel, args::Vararg{Any, N}) where {N} = set_args!(k, 1, args...) -@inline set_args!(k::Kernel, i::Int) = nothing -@inline function set_args!(k::Kernel, i::Int, arg, args::Vararg{Any, N}) where {N} +set_args!(k::Kernel, args::Vararg{Any, N}) where {N} = _set_args!(k, 1, args...) +@inline _set_args!(k::Kernel, i::Int) = nothing +@inline function _set_args!(k::Kernel, i::Int, arg, args::Vararg{Any, N}) where {N} set_arg!(k, i, arg) - return set_args!(k, i + 1, args...) + return _set_args!(k, i + 1, args...) end # work sizes padded to the three dimensions OpenCL devices support