From d148907383e25b3b04b79ef54a79cc445352cf3c Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Sun, 11 Oct 2026 16:26:40 +0200 Subject: [PATCH 1/2] Port to LLVM.jl 10 Backport of the LLVM.jl 10 port on main (3ab40732, part of #1132) to the 2.6 release line, which still uses GPUCompiler 1.x: - Compiler imports `LLVM.IR`, `LLVM.Build` and `LLVM.Passes`; Device imports `LLVM.IR` and `LLVM.Build`. - The GPUCompiler hooks, the device library linker and the LDS zero-initialization use properties instead of accessor functions and setters, `LLVM.before` insertion points and the public floating-point type names. - `create_function`/`call_function` were removed, so the IR generators (indexing, LDS/scratch allocation, random state and Ziggurat tables, device strings, `memcpy!`/`memset!`, the exception CAS) are now `@llvmgenerated` functions generating the same IR. - `get_global_pointer` and `string_length` had no callers, so they are removed instead of ported, as on main. Requires GPUCompiler 1.23.1, the 1.x release ported to LLVM.jl 10. Assisted-by: Claude Code (Opus 5.5) --- Project.toml | 4 +- src/AMDGPU.jl | 2 +- src/compiler/Compiler.jl | 2 +- src/compiler/codegen.jl | 26 +++---- src/compiler/device_libs.jl | 12 +-- src/compiler/zeroinit_lds.jl | 34 ++++----- src/device/Device.jl | 3 +- src/device/gcn/indexing.jl | 109 ++++++++++----------------- src/device/gcn/memory_static.jl | 126 ++++++++++---------------------- src/device/globals.jl | 44 ----------- src/device/random.jl | 98 ++++++++----------------- src/device/runtime.jl | 32 ++------ src/device/strings.jl | 65 +--------------- 13 files changed, 160 insertions(+), 397 deletions(-) delete mode 100644 src/device/globals.jl diff --git a/Project.toml b/Project.toml index 0b0611955..37d30f421 100644 --- a/Project.toml +++ b/Project.toml @@ -60,11 +60,11 @@ ChainRulesCore = "1" EnzymeCore = "0.8" ExprTools = "0.1" GPUArrays = "11.3.1" -GPUCompiler = "1" +GPUCompiler = "1.23.1" GPUToolbox = "0.1.0, 0.2, 0.3, 1, 2, 3" KernelAbstractions = "0.9.2" LLD_jll = "15, 16, 17, 18, 19, 20, 21.1" -LLVM = "9" +LLVM = "10" LLVM_jll = "15, 16, 17, 18, 19, 20, 21.1" PrecompileTools = "1" Preferences = "1" diff --git a/src/AMDGPU.jl b/src/AMDGPU.jl index 59ef0bdf7..21771c281 100644 --- a/src/AMDGPU.jl +++ b/src/AMDGPU.jl @@ -7,7 +7,7 @@ using GPUCompiler using GPUArrays using GPUArrays: allowscalar using Libdl -using LLVM, LLVM.Interop +using LLVM using Preferences using Printf diff --git a/src/compiler/Compiler.jl b/src/compiler/Compiler.jl index 5928e7110..c4f01f0f9 100644 --- a/src/compiler/Compiler.jl +++ b/src/compiler/Compiler.jl @@ -4,7 +4,7 @@ import Core: LLVMPtr import LLD_jll using ..GPUCompiler -using ..LLVM +using ..LLVM, ..LLVM.IR, ..LLVM.Build, ..LLVM.Passes using Printf import ..AMDGPU diff --git a/src/compiler/codegen.jl b/src/compiler/codegen.jl index f859d7a32..5a61414a4 100644 --- a/src/compiler/codegen.jl +++ b/src/compiler/codegen.jl @@ -57,14 +57,14 @@ function GPUCompiler.finish_module!( wavefrontsize64=job.config.params.wavefrontsize64) # Set kernel target cpu and features. - if LLVM.callconv(entry) == LLVM.API.LLVMAMDGPUKERNELCallConv + if entry.callconv == LLVM.API.LLVMAMDGPUKERNELCallConv target_cpu_attr = StringAttribute("target-cpu", job.config.target.dev_isa) target_features_attr = StringAttribute("target-features", job.config.target.features) atomic_attr = StringAttribute("amdgpu-unsafe-fp-atomics", "true") # TODO add convergent, mustprogress, willreturn attributes? - attrs = LLVM.function_attributes(entry) + attrs = entry.function_attributes push!(attrs, target_cpu_attr) push!(attrs, target_features_attr) push!(attrs, atomic_attr) @@ -82,10 +82,10 @@ function GPUCompiler.finish_module!( "signal_exception", "report_exception", "malloc", "__throw_") inline_attr = EnumAttribute("alwaysinline") - for fn in LLVM.functions(mod) - do_inline = any(occursin.(target_fns, LLVM.name(fn))) + for fn in mod.functions + do_inline = any(occursin.(target_fns, fn.name)) if job.config.params.unsafe_fp_atomics || do_inline - attrs = LLVM.function_attributes(fn) + attrs = fn.function_attributes do_inline && inline_attr ∉ collect(attrs) && push!(attrs, inline_attr) @@ -99,13 +99,13 @@ function GPUCompiler.finish_module!( fp_binops = (LLVM.API.LLVMAtomicRMWBinOpFAdd, LLVM.API.LLVMAtomicRMWBinOpFSub, LLVM.API.LLVMAtomicRMWBinOpFMax, LLVM.API.LLVMAtomicRMWBinOpFMin) empty_md = MDNode(Metadata[]) - for fn in LLVM.functions(mod), bb in LLVM.blocks(fn), inst in LLVM.instructions(bb) + for fn in mod.functions, bb in fn.blocks, inst in bb.instructions inst isa LLVM.AtomicRMWInst || continue - op = LLVM.binop(inst) + op = inst.binop op ∈ fp_binops || continue - md = LLVM.metadata(inst) + md = inst.metadata md["amdgpu.no.fine.grained.memory"] = empty_md - if op == LLVM.API.LLVMAtomicRMWBinOpFAdd && LLVM.value_type(inst) == LLVM.FloatType() + if op == LLVM.API.LLVMAtomicRMWBinOpFAdd && inst.value_type isa LLVM.FloatType md["amdgpu.ignore.denormal.mode"] = empty_md end end @@ -208,8 +208,8 @@ function find_global_hostcalls(mod::LLVM.Module) :malloc_hostcall, :free_hostcall, :print_hostcall, :printf_hostcall) global_hostcalls = Symbol[] - for gbl in LLVM.globals(mod), gbl_name in global_hostcall_names - occursin("__$gbl_name", LLVM.name(gbl)) || continue + for gbl in mod.globals, gbl_name in global_hostcall_names + occursin("__$gbl_name", gbl.name) || continue push!(global_hostcalls, gbl_name) end return global_hostcalls @@ -236,8 +236,8 @@ function hipcompile(@nospecialize(job::CompilerJob)) """ end - entry = LLVM.name(meta.entry) - extinit_globals = filter(isextinit, collect(LLVM.globals(meta.ir))) .|> LLVM.name + entry = meta.entry.name + extinit_globals = [gv.name for gv in meta.ir.globals if gv.externally_initialized] if !isempty(extinit_globals) @warn """ HIP backend does not support setting extinit globals. diff --git a/src/compiler/device_libs.jl b/src/compiler/device_libs.jl index 4f01667d8..ad5217b9a 100644 --- a/src/compiler/device_libs.jl +++ b/src/compiler/device_libs.jl @@ -76,17 +76,17 @@ function load_and_link!(devlib::DevLib, mod::LLVM.Module) inline_attr = EnumAttribute("alwaysinline") noinline_attr = EnumAttribute("noinline") - for f in LLVM.functions(lib) - fn_name = LLVM.name(f) + for f in lib.functions + fn_name = f.name # FIXME: We should be able to inline this, that we can't means # we are inserting calls to it late. startswith(fn_name, "__ockl_hsa_signal") && continue - attrs = function_attributes(f) + attrs = f.function_attributes inline = true for attr in collect(attrs) - if kind(attr) == kind(noinline_attr) + if attr.kind == noinline_attr.kind inline = false break end @@ -95,8 +95,8 @@ function load_and_link!(devlib::DevLib, mod::LLVM.Module) end # override triple and datalayout to avoid warnings - triple!(lib, triple(mod)) - datalayout!(lib, datalayout(mod)) + lib.triple = mod.triple + lib.datalayout = mod.datalayout LLVM.link!(mod, lib) return end diff --git a/src/compiler/zeroinit_lds.jl b/src/compiler/zeroinit_lds.jl index 9aa712a56..ef3214427 100644 --- a/src/compiler/zeroinit_lds.jl +++ b/src/compiler/zeroinit_lds.jl @@ -1,31 +1,30 @@ # Calculate the size of an LLVM type. -llvmsize(::LLVM.LLVMHalf) = sizeof(Float16) -llvmsize(::LLVM.LLVMFloat) = sizeof(Float32) -llvmsize(::LLVM.LLVMDouble) = sizeof(Float64) +llvmsize(::LLVM.HalfType) = sizeof(Float16) +llvmsize(::LLVM.FloatType) = sizeof(Float32) +llvmsize(::LLVM.DoubleType) = sizeof(Float64) function llvmsize(::LLVM.IntegerType) - div(Int(intwidth(GenericValue(LLVM.Int128Type(), -1))), 8) + div(Int(LLVM.GenericValue(LLVM.Int128Type(), -1).intwidth), 8) end -llvmsize(ty::LLVM.ArrayType) = length(ty) * llvmsize(eltype(ty)) +llvmsize(ty::LLVM.ArrayType) = ty.length * llvmsize(ty.element_type) llvmsize(ty::LLVM.StructType) = ispacked(ty) ? - sum(llvmsize(elem) for elem in elements(ty)) : - 8 * length(elements(ty)) # FIXME: Properly determine non-packed sizing + sum(llvmsize(elem) for elem in ty.elements) : + 8 * length(ty.elements) # FIXME: Properly determine non-packed sizing llvmsize(ty::LLVM.PointerType) = div(Sys.WORD_SIZE, 8) -llvmsize(ty::LLVM.VectorType) = size(ty) +llvmsize(ty::LLVM.VectorType) = ty.length llvmsize(ty) = error("Unknown size for type: $ty, typeof: $(typeof(ty))") function zeroinit_lds!(mod::LLVM.Module, entry::LLVM.Function) - if LLVM.callconv(entry) != LLVM.API.LLVMAMDGPUKERNELCallConv + if entry.callconv != LLVM.API.LLVMAMDGPUKERNELCallConv return entry end to_init = [] - for gbl in LLVM.globals(mod) - if startswith(LLVM.name(gbl), "__zeroinit") - vt = value_type(gbl) - as = LLVM.addrspace(vt) + for gbl in mod.globals + if startswith(gbl.name, "__zeroinit") + as = gbl.value_type.addrspace if as == AMDGPU.Device.AS.Local - sz = llvmsize(global_value_type(gbl)) + sz = llvmsize(gbl.global_value_type) push!(to_init, (gbl, sz)) end end @@ -34,15 +33,14 @@ function zeroinit_lds!(mod::LLVM.Module, entry::LLVM.Function) @dispose builder=IRBuilder() begin # Make these the first operations we do. - block = first(LLVM.blocks(entry)) - instruction = first(LLVM.instructions(block)) - position!(builder, instruction) + instruction = first(entry.entry.instructions) + position!(builder, LLVM.before(instruction)) # Use memset to clear all values to 0. for (gbl, sz) in to_init sz == 0 && continue LLVM.memset!(builder, gbl, - ConstantInt(UInt8(0)), ConstantInt(sz), LLVM.alignment(gbl)) + ConstantInt(UInt8(0)), ConstantInt(sz), gbl.alignment) end # Synchronize the workgroup to prevent races. diff --git a/src/device/Device.jl b/src/device/Device.jl index 3ac9463ac..c797e36a9 100644 --- a/src/device/Device.jl +++ b/src/device/Device.jl @@ -2,7 +2,7 @@ module Device using ..BFloat16s using ..GPUCompiler -using ..LLVM +using ..LLVM, ..LLVM.IR, ..LLVM.Build using ..LLVM.Interop import ..Adapt @@ -19,7 +19,6 @@ import .AMDGPU: aligned_sizeof import ..UnsafeAtomics include("addrspaces.jl") -include("globals.jl") include("strings.jl") include("exceptions.jl") include("gcn.jl") diff --git a/src/device/gcn/indexing.jl b/src/device/gcn/indexing.jl index 3a24089f2..ede1b712a 100644 --- a/src/device/gcn/indexing.jl +++ b/src/device/gcn/indexing.jl @@ -1,76 +1,47 @@ # Indexing and dimensions -@generated function _index(::Val{fname}, ::Val{name}, ::Val{range}) where {fname, name, range} - @dispose ctx=Context() begin - T_int32 = LLVM.Int32Type() - - # create function - llvm_f, _ = create_function(T_int32) - mod = LLVM.parent(llvm_f) - - # generate IR - @dispose builder=IRBuilder() begin - entry = BasicBlock(llvm_f, "entry") - position!(builder, entry) - - # call the indexing intrinsic - intr_typ = LLVM.FunctionType(T_int32) - intr = LLVM.Function(mod, "llvm.amdgcn.$fname.id.$name", intr_typ) - idx = call!(builder, intr_typ, intr) - - # attach range metadata - metadata(idx)[LLVM.MD_range] = MDNode([ - ConstantInt(UInt32(range.start)), - ConstantInt(UInt32(range.stop))]) - ret!(builder, idx) - end - - call_function(llvm_f, UInt32) - end +@llvmgenerated builder function _index(::Val{fname}, ::Val{name}, ::Val{range})::UInt32 where {fname, name, range} + T_int32 = LLVM.Int32Type() + + # call the indexing intrinsic + intr_typ = LLVM.FunctionType(T_int32) + intr = LLVM.Function(current_module(builder), "llvm.amdgcn.$fname.id.$name", intr_typ) + idx = call!(builder, intr_typ, intr) + + # attach range metadata + idx.metadata[MD_range] = MDNode([ + ConstantInt(UInt32(range.start)), + ConstantInt(UInt32(range.stop))]) + idx end -@generated function _dim(::Val{base}, ::Val{off}, ::Val{range}, ::Type{T}) where {base, off, range, T} - @dispose ctx=Context() begin - T_int8 = LLVM.Int8Type() - T_int32 = LLVM.Int32Type() - - _as = convert(Int, AS.Constant) - T_ptr_i8 = LLVM.PointerType(T_int8, _as) - T_ptr_i32 = LLVM.PointerType(T_int32, _as) - - T_T = convert(LLVMType, T) - T_ptr_T = LLVM.PointerType(T_T, _as) - - # create function - llvm_f, _ = create_function(T_int32) - mod = LLVM.parent(llvm_f) - - # generate IR - @dispose builder=IRBuilder() begin - entry = BasicBlock(llvm_f, "entry") - position!(builder, entry) - - # get the kernel dispatch pointer - intr_typ = LLVM.FunctionType(T_ptr_i8) - intr = LLVM.Function(mod, "llvm.amdgcn.dispatch.ptr", intr_typ) - ptr = call!(builder, intr_typ, intr) - - # load the index - offset = base + ((off - 1) * aligned_sizeof(T)) - idx_ptr_i8 = inbounds_gep!(builder, T_int8, ptr, [ConstantInt(offset)]) - idx_ptr_T = bitcast!(builder, idx_ptr_i8, T_ptr_T) - idx_T = load!(builder, T_T, idx_ptr_T) - idx = zext!(builder, idx_T, T_int32) - - # attach range metadata - metadata(idx_T)[LLVM.MD_range] = MDNode([ - ConstantInt(T(range.start)), - ConstantInt(T(range.stop))]) - ret!(builder, idx) - end - - call_function(llvm_f, UInt32) - end +@llvmgenerated builder function _dim(::Val{base}, ::Val{off}, ::Val{range}, ::Type{T})::UInt32 where {base, off, range, T} + T_int8 = LLVM.Int8Type() + T_int32 = LLVM.Int32Type() + + _as = convert(Int, AS.Constant) + T_ptr_i8 = LLVM.PointerType(T_int8, _as) + + T_T = convert(LLVMType, T) + T_ptr_T = LLVM.PointerType(T_T, _as) + + # get the kernel dispatch pointer + intr_typ = LLVM.FunctionType(T_ptr_i8) + intr = LLVM.Function(current_module(builder), "llvm.amdgcn.dispatch.ptr", intr_typ) + ptr = call!(builder, intr_typ, intr) + + # load the index + offset = base + ((off - 1) * aligned_sizeof(T)) + idx_ptr_i8 = inbounds_gep!(builder, T_int8, ptr, [ConstantInt(offset)]) + idx_ptr_T = bitcast!(builder, idx_ptr_i8, T_ptr_T) + idx_T = load!(builder, T_T, idx_ptr_T) + idx = zext!(builder, idx_T, T_int32) + + # attach range metadata + idx_T.metadata[MD_range] = MDNode([ + ConstantInt(T(range.start)), + ConstantInt(T(range.stop))]) + idx end # TODO: look these up for the current device/queue diff --git a/src/device/gcn/memory_static.jl b/src/device/gcn/memory_static.jl index b57937911..409754679 100644 --- a/src/device/gcn/memory_static.jl +++ b/src/device/gcn/memory_static.jl @@ -1,54 +1,41 @@ "Allocates on-device memory statically from the specified address space." -@generated function alloc_special( +@llvmgenerated builder function alloc_special( ::Val{id}, ::Type{T}, ::Val{as}, ::Val{len}, ::Val{zeroinit} = Val{false}(), -) where {id,T,as,len,zeroinit} - @dispose ctx=Context() begin - eltyp = convert(LLVMType, T) +)::LLVMPtr{T,as} where {id,T,as,len,zeroinit} + eltyp = convert(LLVMType, T) - # old versions of GPUArrays invoke _shmem with an integer id; make sure those are unique - if !isa(id, String) || !isa(id, Symbol) - id = "alloc_special_$id" - end - if zeroinit - id = "__zeroinit_" * id - end - - T_ptr_i8 = convert(LLVMType, LLVMPtr{T,as}) - - # create a function - llvm_f, _ = create_function(T_ptr_i8) + # old versions of GPUArrays invoke _shmem with an integer id; make sure those are unique + name = id + if !isa(id, String) || !isa(id, Symbol) + name = "alloc_special_$id" + end + if zeroinit + name = "__zeroinit_" * name + end - # create the global variable - mod = LLVM.parent(llvm_f) - gv_typ = LLVM.ArrayType(eltyp, len) - gv = GlobalVariable(mod, gv_typ, string(id), as) - if len > 0 - if as == AS.Local - linkage!(gv, LLVM.API.LLVMExternalLinkage) - # NOTE: Backend doesn't support initializer for local AS - elseif as == AS.Private - linkage!(gv, LLVM.API.LLVMInternalLinkage) - initializer!(gv, null(gv_typ)) - end + T_ptr_i8 = convert(LLVMType, LLVMPtr{T,as}) + + # create the global variable + gv_typ = LLVM.ArrayType(eltyp, len) + gv = GlobalVariable(current_module(builder), gv_typ, name, as) + if len > 0 + if as == AS.Local + gv.linkage = LLVM.API.LLVMExternalLinkage + # NOTE: Backend doesn't support initializer for local AS + elseif as == AS.Private + gv.linkage = LLVM.API.LLVMInternalLinkage + gv.initializer = null(gv_typ) end + end - # By requesting a larger-than-datatype alignment, - # we might be able to vectorize. - # TODO: Make the alignment configurable - alignment!(gv, Base.max(32, Base.datatype_alignment(T))) - - # generate IR - @dispose builder=IRBuilder() begin - entry = BasicBlock(llvm_f, "entry") - position!(builder, entry) - - ptr_with_as = gep!(builder, gv_typ, gv, [ConstantInt(0), ConstantInt(0)]) - ptr = bitcast!(builder, ptr_with_as, T_ptr_i8) - ret!(builder, ptr) - end + # By requesting a larger-than-datatype alignment, + # we might be able to vectorize. + # TODO: Make the alignment configurable + gv.alignment = Base.max(32, Base.datatype_alignment(T)) - call_function(llvm_f, LLVMPtr{T,as}) - end + # generate IR + ptr_with_as = gep!(builder, gv_typ, gv, [ConstantInt(0), ConstantInt(0)]) + bitcast!(builder, ptr_with_as, T_ptr_i8) end @inline alloc_local(id, T, len, zeroinit=false) = @@ -95,53 +82,16 @@ macro ROCDynamicLocalArray(T, dims, zeroinit=true, offset=0) end # TODO: Support various types of len -@inline @generated function memcpy!(dest_ptr::LLVMPtr{UInt8,DestAS}, src_ptr::LLVMPtr{UInt8,SrcAS}, len::LT) where {DestAS,SrcAS,LT<:Union{Int64,UInt64}} - @dispose ctx=Context() begin - T_nothing = LLVM.VoidType() - T_pint8_dest = convert(LLVMType, dest_ptr) - T_pint8_src = convert(LLVMType, src_ptr) - T_int64 = convert(LLVMType, len) - T_int1 = LLVM.Int1Type() - - llvm_f, _ = create_function(T_nothing, [T_pint8_dest, T_pint8_src, T_int64]) - mod = LLVM.parent(llvm_f) - T_intr = LLVM.FunctionType(T_nothing, [T_pint8_dest, T_pint8_src, T_int64, T_int1]) - intr = LLVM.Function(mod, "llvm.memcpy.p$(DestAS)i8.p$(SrcAS)i8.i64", T_intr) - @dispose builder=IRBuilder() begin - entry = BasicBlock(llvm_f, "entry") - position!(builder, entry) - - dest_ptr_i8 = parameters(llvm_f)[1] - src_ptr_i8 = parameters(llvm_f)[2] - call!(builder, T_intr, intr, [dest_ptr_i8, src_ptr_i8, parameters(llvm_f)[3], ConstantInt(T_int1, 0)]) - ret!(builder) - end - call_function(llvm_f, Nothing, Tuple{LLVMPtr{UInt8,DestAS},LLVMPtr{UInt8,SrcAS},LT}, :dest_ptr, :src_ptr, :len) - end +# NOTE: these shadow LLVM.Build's `memcpy!` and `memset!`, so use those qualified. +@llvmgenerated builder function memcpy!(dest_ptr::LLVMPtr{UInt8,DestAS}, src_ptr::LLVMPtr{UInt8,SrcAS}, len::LT)::Nothing where {DestAS,SrcAS,LT<:Union{Int64,UInt64}} + LLVM.memcpy!(builder, dest_ptr, src_ptr, len) + nothing end memcpy!(dest_ptr::LLVMPtr{T,DestAS}, src_ptr::LLVMPtr{T,SrcAS}, len::Integer) where {T,DestAS,SrcAS} = memcpy!(reinterpret(LLVMPtr{UInt8,DestAS}, dest_ptr), reinterpret(LLVMPtr{UInt8,SrcAS}, src_ptr), UInt64(len)) -@inline @generated function memset!(dest_ptr::LLVMPtr{UInt8,DestAS}, value::UInt8, len::LT) where {DestAS,LT<:Union{Int64,UInt64}} - @dispose ctx=Context() begin - T_nothing = LLVM.VoidType() - T_pint8_dest = convert(LLVMType, dest_ptr) - T_int8 = convert(LLVMType, value) - T_int64 = convert(LLVMType, len) - T_int1 = LLVM.Int1Type() - - llvm_f, _ = create_function(T_nothing, [T_pint8_dest, T_int8, T_int64]) - mod = LLVM.parent(llvm_f) - T_intr = LLVM.FunctionType(T_nothing, [T_pint8_dest, T_int8, T_int64, T_int1]) - intr = LLVM.Function(mod, "llvm.memset.p$(DestAS)i8.i64", T_intr) - @dispose builder=IRBuilder() begin - entry = BasicBlock(llvm_f, "entry") - position!(builder, entry) - - call!(builder, T_intr, intr, [parameters(llvm_f)[1], parameters(llvm_f)[2], parameters(llvm_f)[3], ConstantInt(T_int1, 0)]) - ret!(builder) - end - call_function(llvm_f, Nothing, Tuple{LLVMPtr{UInt8,DestAS},UInt8,LT}, :dest_ptr, :value, :len) - end +@llvmgenerated builder function memset!(dest_ptr::LLVMPtr{UInt8,DestAS}, value::UInt8, len::LT)::Nothing where {DestAS,LT<:Union{Int64,UInt64}} + LLVM.memset!(builder, dest_ptr, value, len) + nothing end memset!(dest_ptr::LLVMPtr{T,DestAS}, value::UInt8, len::Integer) where {T,DestAS} = memset!(convert(LLVMPtr{UInt8,DestAS}, dest_ptr), value, UInt64(len)) diff --git a/src/device/globals.jl b/src/device/globals.jl deleted file mode 100644 index 125489965..000000000 --- a/src/device/globals.jl +++ /dev/null @@ -1,44 +0,0 @@ -# copied from CUDAnative PR 422 - -# Gets a pointer to a global with a particular name. If the global -# does not exist yet, then it is declared in the global memory address -# space. -@inline @generated function get_global_pointer(::Val{global_name}, ::Type{T})::LLVMPtr{T} where {global_name, T} - @dispose ctx=Context() begin - T_global = convert(LLVMType, T) - T_result = convert(LLVMType, Ptr{T}) - - # Create a thunk that computes a pointer to the global. - llvm_f, _ = create_function(T_result) - mod = LLVM.parent(llvm_f) - - # Figure out if the global has been defined already. - global_set = LLVM.globals(mod) - global_name_string = String(global_name) - if haskey(global_set, global_name_string) - global_var = global_set[global_name_string] - else - # If the global hasn't been defined already, then we'll define - # it in the global address space, i.e., address space one. - global_var = GlobalVariable(mod, T_global, global_name_string, 1) - linkage!(global_var, LLVM.API.LLVMExternalLinkage) - extinit!(global_var, true) - #set_used!(mod, global_var) - end - - # Generate IR that computes the global's address. - @dispose builder=IRBuilder() begin - entry = BasicBlock(llvm_f, "entry") - position!(builder, entry) - - # Cast the global variable's type to the result type. - result = ptrtoint!(builder, global_var, T_result) - ret!(builder, result) - end - - # Call the function. - quote - $(LLVMPtr{T, AS.Global})($(call_function(llvm_f, Ptr{T}))) - end - end -end diff --git a/src/device/random.jl b/src/device/random.jl index cce7d1cbf..cc611e2be 100644 --- a/src/device/random.jl +++ b/src/device/random.jl @@ -9,37 +9,19 @@ import RandomNumbers # global state -@inline @generated function emit_global_random_values(::Val{name}) where name - @dispose ctx=Context() begin - T_val = convert(LLVMType, UInt32) - T_ptr = convert(LLVMType, LLVMPtr{UInt32,AS.Local}) - - # define function and get LLVM module - llvm_f, _ = create_function(T_ptr) - mod = LLVM.parent(llvm_f) - - # create a global memory global variable - T_global = LLVM.ArrayType(T_val, 32) - gv = GlobalVariable(mod, T_global, "__zeroinit_global_random_$(name)", AS.Local) - linkage!(gv, LLVM.API.LLVMExternalLinkage) - - # TODO: we need alwaysinline to ensure we don't access LDS in a non-kernel function - push!(function_attributes(llvm_f), EnumAttribute("alwaysinline")) - - # generate IR - @dispose builder=IRBuilder() begin - entry = BasicBlock(llvm_f, "entry") - position!(builder, entry) - - ptr = gep!(builder, T_global, gv, [ConstantInt(0), ConstantInt(0)]) - - untyped_ptr = bitcast!(builder, ptr, T_ptr) - - ret!(builder, untyped_ptr) - end - - call_function(llvm_f, LLVMPtr{UInt32,AS.Local}) - end +# NOTE: the generated function is `alwaysinline`, which ensures we don't access LDS in a +# non-kernel function +@llvmgenerated builder function emit_global_random_values(::Val{name})::LLVMPtr{UInt32,AS.Local} where name + T_val = convert(LLVMType, UInt32) + T_ptr = convert(LLVMType, LLVMPtr{UInt32,AS.Local}) + + # create a global memory global variable + T_global = LLVM.ArrayType(T_val, 32) + gv = GlobalVariable(current_module(builder), T_global, "__zeroinit_global_random_$(name)", AS.Local) + gv.linkage = LLVM.API.LLVMExternalLinkage + + ptr = gep!(builder, T_global, gv, [ConstantInt(0), ConstantInt(0)]) + bitcast!(builder, ptr, T_ptr) end # shared memory with the actual seed, per warp, loaded lazily or overridden by calling `seed!` @@ -188,48 +170,30 @@ end # copied from Base because we don't support its global tables # a hacky method of exposing constant tables as constant GPU memory -function emit_constant_array(name::Symbol, data::AbstractArray{T}) where {T} - @dispose ctx=Context() begin - T_val = convert(LLVMType, T) - T_ptr = convert(LLVMType, LLVMPtr{T,AS.Constant}) - - # define function and get LLVM module - llvm_f, _ = create_function(T_ptr) - mod = LLVM.parent(llvm_f) - - # create a global memory global variable - # TODO: global_var alignment? - T_global = LLVM.ArrayType(T_val, length(data)) - # XXX: why can't we use a single name like emit_shmem - gv = GlobalVariable(mod, T_global, "gpu_$(name)_data", AS.Constant) - alignment!(gv, 16) - linkage!(gv, LLVM.API.LLVMInternalLinkage) - initializer!(gv, ConstantArray(data)) - - # generate IR - @dispose builder=IRBuilder() begin - entry = BasicBlock(llvm_f, "entry") - position!(builder, entry) - - ptr = gep!(builder, T_global, gv, [ConstantInt(0), ConstantInt(0)]) - - untyped_ptr = bitcast!(builder, ptr, T_ptr) - - ret!(builder, untyped_ptr) - end - - call_function(llvm_f, LLVMPtr{T,AS.Constant}) - end +@llvmgenerated builder function emit_constant_array(::Val{name}, ::Type{T})::LLVMPtr{T,AS.Constant} where {name, T} + data = getfield(Random, name)::AbstractArray{T} + T_val = convert(LLVMType, T) + T_ptr = convert(LLVMType, LLVMPtr{T,AS.Constant}) + + # create a global memory global variable + # TODO: global_var alignment? + T_global = LLVM.ArrayType(T_val, length(data)) + # XXX: why can't we use a single name like emit_shmem + gv = GlobalVariable(current_module(builder), T_global, "gpu_$(name)_data", AS.Constant) + gv.alignment = 16 + gv.linkage = LLVM.API.LLVMInternalLinkage + gv.initializer = ConstantArray(data) + + ptr = gep!(builder, T_global, gv, [ConstantInt(0), ConstantInt(0)]) + bitcast!(builder, ptr, T_ptr) end for var in [:ki, :wi, :fi, :ke, :we, :fe] val = getfield(Random, var) gpu_var = Symbol("gpu_$var") arr_typ = :(ROCDeviceArray{$(eltype(val)),$(ndims(val)),AS.Constant}) - @eval @inline @generated function $gpu_var() - ptr = emit_constant_array($(QuoteNode(var)), $val) - Expr(:call, $arr_typ, $(size(val)), ptr) - end + @eval @inline $gpu_var() = + $arr_typ($(size(val)), emit_constant_array($(Val(var)), $(eltype(val)))) end ## randn diff --git a/src/device/runtime.jl b/src/device/runtime.jl index dcabf27cb..fa3f95e3c 100644 --- a/src/device/runtime.jl +++ b/src/device/runtime.jl @@ -7,31 +7,13 @@ GPUCompiler.reset_runtime() @inline @generated kernel_state() = GPUCompiler.kernel_state_value(AMDGPU.KernelState) -@generated function llvm_atomic_cas(ptr::LLVMPtr{T,A}, cmp::T, val::T) where {T, A} - @dispose ctx=Context() begin - T_val = convert(LLVMType, T) - T_ptr = convert(LLVMType, ptr) - - T_typed_ptr = LLVM.PointerType(T_val, A) - llvm_f, _ = create_function(T_val, [T_ptr, T_val, T_val]) - - @dispose builder=IRBuilder() begin - entry = BasicBlock(llvm_f, "entry") - position!(builder, entry) - - typed_ptr = bitcast!(builder, parameters(llvm_f)[1], T_typed_ptr) - res = atomic_cmpxchg!( - builder, typed_ptr, parameters(llvm_f)[2], - parameters(llvm_f)[3], - LLVM.API.LLVMAtomicOrderingAcquireRelease, - LLVM.API.LLVMAtomicOrderingAcquire, - #=single threaded=# false) - - rv = extract_value!(builder, res, 0) - ret!(builder, rv) - end - call_function(llvm_f, T, Tuple{LLVMPtr{T,A}, T, T}, :ptr, :cmp, :val) - end +@llvmgenerated builder function llvm_atomic_cas(ptr::LLVMPtr{T,A}, cmp::T, val::T)::T where {T, A} + T_typed_ptr = LLVM.PointerType(convert(LLVMType, T), A) + typed_ptr = bitcast!(builder, ptr, T_typed_ptr) + res = atomic_cmpxchg!(builder, typed_ptr, cmp, val, + LLVM.API.LLVMAtomicOrderingAcquireRelease, + LLVM.API.LLVMAtomicOrderingAcquire) + extract_value!(builder, res, 0) end function output_context() diff --git a/src/device/strings.jl b/src/device/strings.jl index af71e2429..5047759f1 100644 --- a/src/device/strings.jl +++ b/src/device/strings.jl @@ -1,66 +1,9 @@ ## Device-side string utilities -@inline @generated function alloc_string(::Val{sym}) where sym - str = String(sym) - @dispose ctx=Context() begin - T_pint8 = LLVM.PointerType(LLVM.Int8Type(), AS.Global) - llvm_f, _ = create_function(T_pint8) - - @dispose builder=IRBuilder() begin - entry = BasicBlock(llvm_f, "entry") - position!(builder, entry) - - str_ptr = globalstring_ptr!(builder, str) - ptr = addrspacecast!(builder, str_ptr, T_pint8) - ret!(builder, ptr) - end - call_function(llvm_f, LLVMPtr{UInt8, AS.Global}) - end -end - -@generated function string_length(ex::Union{Ptr,LLVMPtr}) - @dispose ctx=Context() begin - T_ex = convert(LLVMType, ex) - T_ex_ptr = LLVM.PointerType(T_ex) - T_i8 = LLVM.Int8Type() - T_i8_ptr = LLVM.PointerType(T_i8) - T_i64 = LLVM.Int64Type() - llvm_f, _ = create_function(T_i64, [T_ex]) - mod = LLVM.parent(llvm_f) - - @dispose builder=IRBuilder() begin - entry = BasicBlock(llvm_f, "entry") - check = BasicBlock(llvm_f, "check") - done = BasicBlock(llvm_f, "done") - - position!(builder, entry) - init_offset = ConstantInt(0) - input_ptr = if T_ex isa LLVM.PointerType - parameters(llvm_f)[1] - else - inttoptr!(builder, parameters(llvm_f)[1], T_ex_ptr) - end - if LLVM.addrspace(value_type(input_ptr)) != LLVM.addrspace(T_ex_ptr) - input_ptr = addrspacecast!(builder, input_ptr, T_ex_ptr) - end - input_ptr = bitcast!(builder, input_ptr, T_i8_ptr) - br!(builder, check) - - position!(builder, check) - offset = phi!(builder, T_i64) - next_offset = add!(builder, offset, ConstantInt(1)) - append!(LLVM.incoming(offset), [(init_offset, entry), (next_offset, check)]) - - ptr = gep!(builder, T_i8, input_ptr, [offset]) - value = load!(builder, T_i8, ptr) - cond = icmp!(builder, LLVM.API.LLVMIntEQ, value, ConstantInt(0x0)) - br!(builder, cond, done, check) - - position!(builder, done) - ret!(builder, offset) - end - call_function(llvm_f, Int64, Tuple{ex}, :ex) - end +@llvmgenerated builder function alloc_string(::Val{sym})::LLVMPtr{UInt8,AS.Global} where sym + T_pint8 = LLVM.PointerType(LLVM.Int8Type(), AS.Global) + str_ptr = globalstring_ptr!(builder, String(sym)) + addrspacecast!(builder, str_ptr, T_pint8) end @inline strlen(::Val{S}) where S = length(String(S)) From 96c007fd1e20b9b2404afc31dd2c08b82fe7b2f7 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Sun, 11 Oct 2026 18:30:21 +0200 Subject: [PATCH 2/2] Bump version to 2.6.1 Assisted-by: Claude Code (Opus 5.5) --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 37d30f421..ddbf30c3b 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "AMDGPU" uuid = "21141c5a-9bdb-4563-92ae-f87d6854732e" -version = "2.6.0" +version = "2.6.1" authors = ["Julian P Samaroo ", "Valentin Churavy ", "Anton Smirnov "] [workspace]