[Draft, for CI] Generate code with JuliaCodegen.jl, as an opt-in

.ci/juliacodegen.jl

@@ -0,0 +1,178 @@
+# TEMPORARY: sets up CI environments with the unreleased LLVM.jl, JuliaIR.jl and
+# JuliaCodegen.jl that the `codegen=:juliacodegen` option needs, until those are released.
+# Drop this file, and its uses in `.github/workflows/Test.yml`, once GPUCompiler can depend
+# on registered versions.
+#
+# LLVM.jl's branch comes from `[sources]` in `Project.toml`; it is checked out (to `LLVM.jl/`
+# in this repository) and developed explicitly rather than through `[sources]`, because its
+# LLVMExtra library has to be built from the same checkout, and because Julia 1.10 ignores
+# `[sources]`. JuliaCodegen.jl and JuliaIR.jl (which it depends on) come from the branches
+# below; they are added to the test environment only on Julia 1.12 and later, which
+# JuliaCodegen.jl requires.
+#
+#     julia .ci/juliacodegen.jl build
+#
+# Checks out LLVM.jl and builds its LLVMExtra library if the checkout has wrapper changes
+# that no LLVMExtra_jll release has (LLVM.jl's `deps/build_ci.jl`). Needs git, cmake (or
+# downloads CMake_jll) and a C++ compiler; on Windows, run it in an MSYS2 shell.
+#
+#     julia .ci/juliacodegen.jl test (julia|juliacodegen)
+#
+# Prepares GPUCompiler's test environment (`test/`), for `julia --project=test
+# test/runtests.jl`, with the `codegen` preference set to the given code generator. Also
+# develops LLVM.jl into the default environment (`@v#.#`), with its preferences.
+#
+#     julia .ci/juliacodegen.jl check-ledger FILE
+#
+# Fails unless FILE, the ledger of the JuliaCodegenExt extension (`JULIACODEGEN_LEDGER`)
+# of a run with JuliaCodegen.jl, shows that it compiled code, and reports how much of it
+# fell back to native code.
+
+using Pkg, TOML
+
+const ROOT = dirname(@__DIR__)
+const LLVM_DIR = joinpath(ROOT, "LLVM.jl")
+
+# the branches of the unreleased packages
+const JULIACODEGEN = (; url="https://github.com/JuliaLLVM/JuliaCodegen.jl", rev="vc/jc-24-gpu-gaps")
+const JULIAIR = (; url="https://github.com/JuliaLLVM/JuliaIR.jl", rev="vc/runtime-cfunction")
+
+function source(project, name)
+    toml = TOML.parsefile(joinpath(ROOT, project, "Project.toml"))
+    return toml["sources"][name]
+end
+
+# remove the `[sources]` entries of the packages this script develops, which Pkg 1.11
+# refuses to replace ("`path` and `url` are conflicting")
+function strip_sources(project)
+    file = joinpath(ROOT, project, "Project.toml")
+    toml = TOML.parsefile(file)
+    sources = get(toml, "sources", Dict())
+    delete!(sources, "LLVM")
+    isempty(sources) && delete!(toml, "sources")
+    open(io -> TOML.print(io, toml), file, "w")
+end
+
+run_git(args) = run(`git $args`)
+
+function build()
+    llvm = source(".", "LLVM")
+    if !isdir(LLVM_DIR)
+        # the full history, as build_ci.jl compares it with LLVMExtra_jll's releases
+        run_git(`clone --quiet --branch $(llvm["rev"]) $(llvm["url"]) $LLVM_DIR`)
+    end
+    run_git(`-C $LLVM_DIR log -1 --format='LLVM.jl %H %s'`)
+    # left by a build for another Julia version, which build_ci.jl may not replace
+    rm(joinpath(LLVM_DIR, "LocalPreferences.toml"); force=true)
+    Pkg.activate(LLVM_DIR)
+    Pkg.instantiate()
+
+    if Sys.which("cmake") === nothing
+        # a wrapper on PATH, as build_local.jl looks for `cmake` there
+        Pkg.activate(; temp=true)
+        Pkg.add("CMake_jll")
+        @eval using CMake_jll
+        bin = mktempdir()
+        libpath = Base.invokelatest(() -> CMake_jll.LIBPATH[])
+        exe = Base.invokelatest(() -> CMake_jll.cmake_path)
+        write(joinpath(bin, "cmake"), """
+            #!/bin/sh
+            LD_LIBRARY_PATH="$libpath:\$LD_LIBRARY_PATH" exec "$exe" "\$@"
+            """)
+        chmod(joinpath(bin, "cmake"), 0o755)
+        ENV["PATH"] = bin * (Sys.iswindows() ? ";" : ":") * ENV["PATH"]
+    end
+
+    run(`$(Base.julia_cmd()) --project=$(joinpath(LLVM_DIR, "deps")) $(joinpath(LLVM_DIR, "deps", "build_ci.jl"))`)
+    prefs = joinpath(LLVM_DIR, "LocalPreferences.toml")
+    if isfile(prefs)
+        print(read(prefs, String))
+    else
+        println("Using LLVMExtra_jll")
+    end
+end
+
+# the preferences for an environment: LLVM.jl's LLVMExtra build, if any, and the code
+# generator
+function write_preferences(dir, mode)
+    prefs = Dict{String,Any}()
+    llvm_prefs = joinpath(LLVM_DIR, "LocalPreferences.toml")
+    isfile(llvm_prefs) && merge!(prefs, TOML.parsefile(llvm_prefs))
+    prefs["GPUCompiler"] = Dict("codegen" => mode)
+    file = joinpath(dir, "LocalPreferences.toml")
+    open(io -> TOML.print(io, prefs), file, "w")
+    println("$file:")
+    print(read(file, String))
+end
+
+function check_mode(mode)
+    mode in ("julia", "juliacodegen") ||
+        error("Unknown code generator $mode; expected julia or juliacodegen")
+    mode == "juliacodegen" && VERSION < v"1.12" &&
+        error("JuliaCodegen.jl requires Julia 1.12 or later")
+end
+
+function test(mode)
+    check_mode(mode)
+    isdir(LLVM_DIR) || error("Run `julia .ci/juliacodegen.jl build` first")
+    strip_sources(".")
+    write_preferences(joinpath(ROOT, "test"), mode)
+
+    Pkg.activate(joinpath(ROOT, "test"))
+    if VERSION >= v"1.12"
+        # first, as they are not registered: any other operation would need to resolve them
+        Pkg.add([PackageSpec(; JULIAIR...), PackageSpec(; JULIACODEGEN...)])
+    end
+    specs = [PackageSpec(path=LLVM_DIR)]
+    # Julia 1.10 and 1.11 have no workspaces, so the test environment needs GPUCompiler
+    VERSION < v"1.12" && pushfirst!(specs, PackageSpec(path=ROOT))
+    Pkg.develop(specs)
+    Pkg.status()
+
+    # Aqua's persistent-task check loads GPUCompiler in a new environment, stacked only on
+    # the default one, so that needs LLVM.jl and the preference of its LLVMExtra too
+    Pkg.activate()
+    Pkg.develop(path=LLVM_DIR)
+    write_preferences(dirname(Base.active_project()), mode)
+end
+
+function check_ledger(file)
+    isfile(file) || error("No JuliaCodegen.jl ledger at $file: it did not run")
+    jobs = 0
+    functions = 0
+    fallbacks = 0
+    pids = Set{String}()
+    for (i, line) in enumerate(eachline(file))
+        # PID, "juliacodegen", the method instance, the functions, the fallbacks
+        fields = split(line, '\t')
+        if !(length(fields) == 5 && all(isdigit, fields[1]) && fields[2] == "juliacodegen" &&
+             all(isdigit, fields[4]) && all(isdigit, fields[5]))
+            error("Malformed line $i in the ledger at $file: $(repr(line))")
+        end
+        jobs += 1
+        functions += parse(Int, fields[4])
+        fallbacks += parse(Int, fields[5])
+        push!(pids, fields[1])
+    end
+    println("$jobs compilations of $functions functions, $fallbacks of which call native code, in $(length(pids)) processes")
+    jobs > 0 || error("JuliaCodegen.jl compiled nothing; is GPUCompiler using Julia's code generator?")
+end
+
+function main(args)
+    # the environments are only complete at the end, e.g., JuliaIR.jl fails to precompile
+    # with the registered LLVM.jl that is used until the checkout is developed
+    ENV["JULIA_PKG_PRECOMPILE_AUTO"] = "0"
+    isempty(args) && error("Usage: julia .ci/juliacodegen.jl (build | test MODE | check-ledger FILE)")
+    command = args[1]
+    if command == "build"
+        build()
+    elseif command == "test"
+        test(args[2])
+    elseif command == "check-ledger"
+        check_ledger(args[2])
+    else
+        error("Unknown command $command")
+    end
+end
+
+main(ARGS)

.github/workflows/Test.yml

@@ -13,42 +13,60 @@ concurrency:
 
 jobs:
   release_test:
-    name: Julia ${{ matrix.version }} ${{ matrix.llvm_args }} - ${{ matrix.os }}
+    name: Julia ${{ matrix.version }} ${{ matrix.llvm_args }} - ${{ matrix.os }} - ${{ matrix.codegen }}
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 20
+    # TEMPORARY: raised from 20 for building LLVMExtra
+    timeout-minutes: 45
+    # JuliaCodegen.jl is experimental: its jobs report, and don't fail the workflow
+    continue-on-error: ${{ matrix.codegen == 'juliacodegen' }}
     strategy:
       fail-fast: false
       matrix:
         version: ['1.10', '1.11', '1.12', '1.13', 'nightly']
         os: [ubuntu-24.04, ubuntu-24.04-arm, macOS-15, windows-2025]
         llvm_args: ['']
+        # the code generator (the `codegen` preference); JuliaCodegen.jl needs Julia 1.12
+        codegen: [julia, juliacodegen]
+        exclude:
+          - version: '1.10'
+            codegen: juliacodegen
+          - version: '1.11'
+            codegen: juliacodegen
         include:
           # starting with Julia 1.10, we can enable opaque pointers
           # from Julia 1.12 on, this is the default.
           - version: '1.10'
             os: 'ubuntu-24.04'
             llvm_args: '--opaque-pointers'
+            codegen: julia
           - version: '1.10'
             os: 'ubuntu-24.04-arm'
             llvm_args: '--opaque-pointers'
+            codegen: julia
           - version: '1.10'
             os: 'macOS-15'
             llvm_args: '--opaque-pointers'
+            codegen: julia
           - version: '1.10'
             os: 'windows-2025'
             llvm_args: '--opaque-pointers'
+            codegen: julia
           - version: '1.11'
             os: 'ubuntu-24.04'
             llvm_args: '--opaque-pointers'
+            codegen: julia
           - version: '1.11'
             os: 'ubuntu-24.04-arm'
             llvm_args: '--opaque-pointers'
+            codegen: julia
           - version: '1.11'
             os: 'macOS-15'
             llvm_args: '--opaque-pointers'
+            codegen: julia
           - version: '1.11'
             os: 'windows-2025'
             llvm_args: '--opaque-pointers'
+            codegen: julia
     steps:
       - uses: actions/checkout@v7
 
@@ -60,19 +78,35 @@ jobs:
       - name: Set-up cache
         uses: julia-actions/cache@v3
 
+      # TEMPORARY, until the LLVM.jl, JuliaIR.jl and JuliaCodegen.jl changes this branch
+      # needs are released: build LLVMExtra from the LLVM.jl branch, and set up the test
+      # environment with it and JuliaCodegen.jl (see .ci/juliacodegen.jl). Restore
+      # julia-runtest and the coverage steps after.
+      - uses: msys2/setup-msys2@v2
+        if: runner.os == 'Windows'
+        with:
+          path-type: inherit
+          install: make mingw-w64-x86_64-gcc mingw-w64-x86_64-cmake
+      - name: Build LLVMExtra
+        if: runner.os != 'Windows'
+        run: julia --color=yes .ci/juliacodegen.jl build
+      - name: Build LLVMExtra (in msys2)
+        if: runner.os == 'Windows'
+        shell: msys2 {0}
+        run: julia --color=yes .ci/juliacodegen.jl build
+      - name: Set up the test environment
+        run: julia --color=yes .ci/juliacodegen.jl test ${{ matrix.codegen }}
+
       - name: Run tests
-        uses: julia-actions/julia-runtest@v1
+        run: julia --color=yes --project=test test/runtests.jl
         env:
           JULIA_LLVM_ARGS: ${{ matrix.llvm_args }}
+          JULIACODEGEN_LEDGER: ${{ runner.temp }}/juliacodegen-ledger.tsv
 
-      - name: Process coverage
-        # CoverageTools 1.4.1 uses a PkgLoadSpec field renamed in Julia 1.14.
-        if: matrix.version != 'nightly'
-        uses: julia-actions/julia-processcoverage@v1
-      - name: Submit coverage
-        if: matrix.version != 'nightly'
-        uses: codecov/codecov-action@v7
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          fail_ci_if_error: false
-          files: lcov.info
+      # the suite also runs JuliaCodegen.jl explicitly, so check the preference too
+      - name: Check that JuliaCodegen.jl was the default
+        if: matrix.codegen == 'juliacodegen'
+        run: julia --project=test -e 'using GPUCompiler; GPUCompiler.CODEGEN === :juliacodegen || error("codegen preference not applied")'
+      - name: Check that JuliaCodegen.jl ran
+        if: matrix.codegen == 'juliacodegen'
+        run: julia .ci/juliacodegen.jl check-ledger '${{ runner.temp }}/juliacodegen-ledger.tsv'

.gitignore

@@ -2,3 +2,4 @@ test/Manifest.toml
 test/LocalPreferences.toml
 Manifest.toml
 Manifest-*.toml
+LLVM.jl/

Project.toml

@@ -29,14 +29,19 @@ tree_sitter_spirv_jll = "f0e86581-c468-54df-a4de-3266e11a3c86"
 AMDGPU_LLVM_Backend_jll = "cc5c0156-bd05-5a77-8a68-bb0aafb29019"
 Cthulhu = "f68482b8-f384-11e8-15f7-abe071a5a75f"
 Highlights = "eafb193a-b7ab-5a9e-9068-77385905fa72"
+JuliaCodegen = "f68e5268-cafc-4116-b772-1c15436a0999"
 LLVMDowngrader_jll = "f52de702-fb25-5922-94ba-81dd59b07444"
 NVPTX_LLVM_Backend_jll = "ef6e0fe3-e6ef-59c0-bde6-4989574699e0"
 SPIRV_LLVM_Backend_jll = "4376b9bf-cff8-51b6-bb48-39421dff0d0c"
 SPIRV_LLVM_Translator_jll = "4a5d46fc-d8cf-5151-a261-86b458210efb"
 
+[sources]
+LLVM = {url = "https://github.com/JuliaLLVM/LLVM.jl", rev = "tb/inline-api"}
+
 [extensions]
 CthulhuExt = "Cthulhu"
 HighlightsExt = "Highlights"
+JuliaCodegenExt = "JuliaCodegen"
 
 [compat]
 AMDGPU_LLVM_Backend_jll = "23.1.3"
@@ -45,6 +50,7 @@ Cthulhu = "2, 3"
 ExprTools = "0.1"
 Highlights = "0.6"
 InteractiveUtils = "1"
+JuliaCodegen = "0.1"
 LLVM = "10"
 LLVMDowngrader_jll = "0.11, 0.12"
 Libdl = "1"

README.md

@@ -51,3 +51,27 @@ last commit of a pull request:
 Back-ends are named `cuda`, `amdgpu`, `metal`, `oneapi` and `opencl` (all lowercase), or by
 their target: `ptx` for CUDA.jl, `gcn` for AMDGPU.jl, and `spirv` for both oneAPI.jl and
 OpenCL.jl.
+
+
+## JuliaCodegen.jl (experimental)
+
+GPUCompiler lowers Julia IR to LLVM IR with Julia's own code generator. It can instead use
+[JuliaCodegen.jl](https://github.com/JuliaLLVM/JuliaCodegen.jl), a code generator written
+in Julia, on Julia 1.12 and later. This is selected per compilation with the `codegen`
+keyword argument of `CompilerConfig` (`:julia` or `:juliacodegen`), whose default comes
+from a preference:
+
+```julia
+using Preferences, GPUCompiler
+set_preferences!(GPUCompiler, "codegen" => "juliacodegen")
+```
+
+The preference takes effect after restarting Julia. While precompiling packages that do not
+load JuliaCodegen.jl, configurations keep using Julia's code generator.
+
+JuliaCodegen.jl must be loaded. When the preference selects it, loading GPUCompiler also
+loads JuliaCodegen.jl if the active environment provides it; otherwise, load it with
+`using JuliaCodegen`. Code it does not support yet calls native code, which validation
+reports on targets without Julia's runtime.
+
+To run the test suite with JuliaCodegen.jl, set the preference in `test/LocalPreferences.toml`.

deps/build.jl

@@ -0,0 +1,12 @@
+# TEMPORARY, with .ci/juliacodegen.jl: the assert jobs run TestAsserts.yml from the default
+# branch (it is triggered by `workflow_run`), which only runs julia-buildpkg and
+# julia-runtest. Pkg.build runs this file, so set up the environments here the way
+# Test.yml does, with LLVMExtra built from the LLVM.jl branch. Delete with
+# .ci/juliacodegen.jl.
+if get(ENV, "GITHUB_WORKFLOW", "") == "TestAsserts"
+    script = joinpath(dirname(@__DIR__), ".ci", "juliacodegen.jl")
+    # not in Pkg.build's sandbox environment
+    julia = addenv(Base.julia_cmd(), "JULIA_PROJECT" => nothing, "JULIA_LOAD_PATH" => nothing)
+    run(`$julia --color=yes $script build`)
+    run(`$julia --color=yes $script test $(VERSION >= v"1.12" ? "juliacodegen" : "julia")`)
+end