feat: Intel GPU Max (Ponte Vecchio) OpenMP target offload support

CMakeLists.txt

@@ -31,6 +31,8 @@ option(MFC_DOCUMENTATION "Build documentation"                               OFF
 option(MFC_ALL           "Build everything"                                  OFF)
 option(MFC_SINGLE_PRECISION "Build single precision"                         OFF)
 option(MFC_MIXED_PRECISION "Build mixed precision"                           OFF)
+option(MFC_Intel_AOT       "Build Intel GPU with AOT (spir64_gen) instead of JIT (spir64)" OFF)
+set(MFC_Intel_AOT_DEVICE "pvc" CACHE STRING "ocloc device target for Intel AOT compilation (e.g. pvc, 0xbda)")
 
 if (MFC_ALL)
     set(MFC_PRE_PROCESS   ON FORCE)
@@ -244,6 +246,15 @@ elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Flang")
 elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Intel")
     add_compile_options($<$<COMPILE_LANGUAGE:Fortran>:-free>)
 
+    if (CMAKE_BUILD_TYPE STREQUAL "Debug")
+        add_compile_options(-g -Og -traceback -debug -check all)
+    elseif (CMAKE_BUILD_TYPE STREQUAL "RelDebug")
+        add_compile_options(-g -Og -traceback -check bounds)
+    endif()
+elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "IntelLLVM")
+    add_compile_options($<$<COMPILE_LANGUAGE:Fortran>:-free>
+                        $<$<COMPILE_LANGUAGE:Fortran>:-fpp>)
+
     if (CMAKE_BUILD_TYPE STREQUAL "Debug")
         add_compile_options(-g -Og -traceback -debug -check all)
     elseif (CMAKE_BUILD_TYPE STREQUAL "RelDebug")
@@ -595,6 +606,73 @@ exit 0
                             HINTS "$ENV{OLCF_AFAR_ROOT}/lib" REQUIRED)
                         target_link_libraries(${a_target} PRIVATE ${HIPFFT_LIB})
                     endif()
+                elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "IntelLLVM")
+                    # Intel GPU: oneMKL DFTI with !$omp dispatch for GPU FFT.
+                    # Requires MKLROOT to be set (via oneAPI module or env).
+                    if (NOT DEFINED ENV{MKLROOT})
+                        message(FATAL_ERROR "MKLROOT is not set. Load oneAPI MKL module before building.")
+                    endif()
+                    # Compile mkl_dfti_omp_offload.f90 in isolation with minimal flags.
+                    # The file uses !$omp declare variant with need_device_addr (OpenMP 5.2)
+                    # which requires the global -free -fpp flags to be absent so the
+                    # compiler parses it in standard fixed/free detection mode only.
+                    set(_mkl_omp_src "$ENV{MKLROOT}/include/mkl_dfti_omp_offload.f90")
+                    if (NOT EXISTS "${_mkl_omp_src}")
+                        message(FATAL_ERROR "mkl_dfti_omp_offload.f90 not found in $ENV{MKLROOT}/include")
+                    endif()
+                    set(_mkl_omp_mod_dir "${CMAKE_CURRENT_BINARY_DIR}/mkl_omp_mods")
+                    set(_mkl_omp_obj "${_mkl_omp_mod_dir}/mkl_dfti_omp_offload.o")
+                    set(_mkl_omp_obj_host "${_mkl_omp_mod_dir}/mkl_dfti_omp_offload_host.o")
+                    file(MAKE_DIRECTORY "${_mkl_omp_mod_dir}")
+                    # Find clang-offload-bundler (in ifx's bin/compiler/ subdir).
+                    # CMAKE_Fortran_COMPILER may be an MPI wrapper (mpiifx); resolve the
+                    # underlying ifx from PATH so the HINTS point to the right directory.
+                    find_program(_IFX_REAL ifx REQUIRED)
+                    cmake_path(GET _IFX_REAL PARENT_PATH _ifx_bin)
+                    find_program(CLANG_OFFLOAD_BUNDLER
+                        NAMES clang-offload-bundler
+                        HINTS "${_ifx_bin}/compiler" "${_ifx_bin}" "${_ifx_bin}-llvm"
+                        REQUIRED)
+                    add_custom_command(
+                        OUTPUT  "${_mkl_omp_obj}"
+                                "${_mkl_omp_mod_dir}/mkl_dfti_omp_offload.mod"
+                        COMMAND "${CMAKE_Fortran_COMPILER}"
+                                -fiopenmp -fopenmp-targets=spir64
+                                -c -I"$ENV{MKLROOT}/include"
+                                "${_mkl_omp_src}"
+                                -o "${_mkl_omp_obj}"
+                        WORKING_DIRECTORY "${_mkl_omp_mod_dir}"
+                        DEPENDS "${_mkl_omp_src}"
+                        COMMENT "Compiling MKL OMP offload Fortran module (mkl_dfti_omp_offload)"
+                    )
+                    # Strip the SPIR-V device bundle so the linked object has only host code.
+                    # The SPIR-V contains Import declarations for MKL SYCL DFT functions that
+                    # the OpenMP Level Zero plugin cannot resolve, causing zeModuleDynamicLink
+                    # failure. With host-only code, !$omp dispatch falls back to CPU for DFT.
+                    add_custom_command(
+                        OUTPUT  "${_mkl_omp_obj_host}"
+                        COMMAND "${CLANG_OFFLOAD_BUNDLER}"
+                                --unbundle --type=o
+                                --targets=host-x86_64-unknown-linux-gnu
+                                --input="${_mkl_omp_obj}"
+                                --output="${_mkl_omp_obj_host}"
+                        DEPENDS "${_mkl_omp_obj}"
+                        COMMENT "Stripping SPIR-V from MKL DFT object (host-only, fixes Level Zero link)"
+                    )
+                    add_custom_target(mkl_omp_offload_mod_${a_target}
+                        DEPENDS "${_mkl_omp_obj_host}")
+                    add_dependencies(${a_target} mkl_omp_offload_mod_${a_target})
+                    target_include_directories(${a_target} PRIVATE
+                        "$ENV{MKLROOT}/include" "${_mkl_omp_mod_dir}")
+                    target_link_libraries(${a_target} PRIVATE "${_mkl_omp_obj_host}")
+                    # Link MKL threading + core + SYCL DFT backend
+                    target_link_options(${a_target} PRIVATE -qmkl=parallel)
+                    # mkl_sycl_dft is the name in MKL >= 2023.2; older versions use monolithic mkl_sycl
+                    find_library(MKL_SYCL_DFT NAMES mkl_sycl_dft mkl_sycl
+                        HINTS "$ENV{MKLROOT}/lib" "$ENV{MKLROOT}/lib/intel64" REQUIRED)
+                    find_library(SYCL_LIB sycl HINTS ENV LIBRARY_PATH "${_ifx_bin}/../lib" REQUIRED)
+                    find_library(OPENCL_LIB OpenCL HINTS ENV LIBRARY_PATH "${_ifx_bin}/../lib" REQUIRED)
+                    target_link_libraries(${a_target} PRIVATE ${MKL_SYCL_DFT} ${SYCL_LIB} ${OPENCL_LIB})
                 else()
                     find_package(hipfort COMPONENTS hipfft CONFIG REQUIRED)
                     target_link_libraries(${a_target} PRIVATE hipfort::hipfft)
@@ -636,9 +714,22 @@ exit 0
                     target_compile_options(${a_target} PRIVATE "-mp=gpu" "-Minfo=mp")
                     target_link_options(${a_target} PRIVATE "-mp=gpu")
                     set_target_properties(${a_target} PROPERTIES Fortran_FLAGS "-mp=gpu -gpu=ccall")
-                elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "Intel")
-                    target_compile_options(${a_target} PRIVATE -fopenmp -fopenmp-targets=spir64)
-                    target_link_options(${a_target} PRIVATE -fopenmp -fopenmp-targets=spir64)
+                elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "IntelLLVM")
+                    # Intel GPU: OpenMP target offload to SPIR64 (Xe-HPC / Ponte Vecchio).
+                    # GPU FFT uses oneMKL DFTI via the OpenMP dispatch construct.
+                    if (MFC_Intel_AOT)
+                        # AOT: compile to native ISA via ocloc for Intel GPU Max (pvc).
+                        # Avoids JIT zeModuleCreate failures at runtime on Level Zero.
+                        # SHELL: prevents CMake deduplication and preserves the quoted
+                        # "-device pvc" as a single argument to -Xopenmp-target-backend.
+                        target_compile_options(${a_target} PRIVATE -fiopenmp -fopenmp-targets=spir64_gen
+                            "SHELL:-Xopenmp-target-backend \"-device ${MFC_Intel_AOT_DEVICE}\"")
+                        target_link_options(${a_target} PRIVATE -fiopenmp -fopenmp-targets=spir64_gen
+                            "SHELL:-Xopenmp-target-backend \"-device ${MFC_Intel_AOT_DEVICE}\"")
+                    else()
+                        target_compile_options(${a_target} PRIVATE -fiopenmp -fopenmp-targets=spir64)
+                        target_link_options(${a_target} PRIVATE -fiopenmp -fopenmp-targets=spir64)
+                    endif()
                 elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
                     target_compile_options(${a_target} PRIVATE -fopenmp)
                     target_link_options(${a_target} PRIVATE -fopenmp)

build_intel_gpu.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+#SBATCH -p pvc
+#SBATCH -N 1
+#SBATCH --gres=gpu:pvc:1
+#SBATCH -t 1:30:00
+#SBATCH --mem=32G
+#SBATCH -o /scratch/user/u.sb27915/MFC-intel/build_intel_gpu.log
+#SBATCH -e /scratch/user/u.sb27915/MFC-intel/build_intel_gpu.log
+#SBATCH -J mfc-intel-gpu-build
+
+source /etc/profile
+module load iimpi/2025a imkl/2025.1.0 CMake/3.31.3 Python/3.13.1
+export I_MPI_F90=ifx FC=mpif90
+export UV_CACHE_DIR=/scratch/user/u.sb27915/.cache/uv
+export RUSTUP_HOME=/scratch/user/u.sb27915/.rustup
+export CARGO_HOME=/scratch/user/u.sb27915/.cargo
+export PATH=$CARGO_HOME/bin:$PATH
+
+cd /scratch/user/u.sb27915/MFC-intel
+./mfc.sh build -t pre_process simulation --gpu mp --intel-aot -j 8