Sync with Microsoft ONNX Runtime - 04092026

.github/workflows/linux_cuda_no_cudnn.yml

@@ -5,6 +5,7 @@ on:
     branches: [main, 'rel-*']
     paths:
       - '.github/workflows/linux_cuda_no_cudnn.yml'
+      - 'cmake/onnxruntime_cuda_cccl.cmake'
       - 'cmake/onnxruntime_providers_cuda.cmake'
       - 'cmake/onnxruntime_python.cmake'
       - 'onnxruntime/__init__.py'

.github/workflows/windows_cuda_no_cudnn.yml

@@ -5,6 +5,7 @@ on:
     branches: [main, 'rel-*']
     paths:
       - '.github/workflows/windows_cuda_no_cudnn.yml'
+      - 'cmake/onnxruntime_cuda_cccl.cmake'
       - 'cmake/onnxruntime_providers_cuda.cmake'
       - 'cmake/onnxruntime_providers_cuda_plugin.cmake'
       - 'cmake/onnxruntime_python.cmake'

cmake/deps.txt

@@ -50,7 +50,7 @@ protoc_mac_universal;https://github.com/protocolbuffers/protobuf/releases/downlo
 psimd;https://github.com/Maratyszcza/psimd/archive/072586a71b55b7f8c584153d223e95687148a900.zip;1f5454b01f06f9656b77e4a5e2e31d7422487013
 pthreadpool;https://github.com/google/pthreadpool/archive/dcc9f28589066af0dbd4555579281230abbf74dd.zip;533a77943203ef15ca608bcd9dbe2c94da7451d2
 pybind11;https://github.com/pybind/pybind11/archive/refs/tags/v3.0.2.zip;a064e663b4d7a337ac291d1bef7337ef4e60a1ae
-pytorch_cpuinfo;https://github.com/pytorch/cpuinfo/archive/4628dc060ce4e82345dc166bbac875609db4ff69.zip;e58d4b47c16a982111c897e669ae4f1821a393d7
+pytorch_cpuinfo;https://github.com/pytorch/cpuinfo/archive/66ee79c038d70dad9f08705b2c9b3e58f6d8f512.zip;6e2be17c110e4f4184b059303c722b3c5bd080a6
 re2;https://github.com/google/re2/archive/refs/tags/2024-07-02.zip;646e1728269cde7fcef990bf4a8e87b047882e88
 safeint;https://github.com/dcleblanc/SafeInt/archive/refs/tags/3.0.28.zip;23f252040ff6cb9f1fd18575b32fa8fb5928daac
 tensorboard;https://github.com/tensorflow/tensorboard/archive/373eb09e4c5d2b3cc2493f0949dc4be6b6a45e81.zip;67b833913605a4f3f499894ab11528a702c2b381

cmake/external/onnxruntime_external_deps.cmake

@@ -363,42 +363,52 @@ if (CPUINFO_SUPPORTED)
   set(CPUINFO_BUILD_UNIT_TESTS OFF CACHE INTERNAL "")
   set(CPUINFO_BUILD_MOCK_TESTS OFF CACHE INTERNAL "")
   set(CPUINFO_BUILD_BENCHMARKS OFF CACHE INTERNAL "")
-  if (onnxruntime_target_platform STREQUAL "ARM64EC" OR onnxruntime_target_platform STREQUAL "ARM64")
-    message(STATUS "Applying patches for Windows ARM64/ARM64EC in cpuinfo")
-    onnxruntime_fetchcontent_declare(
-      pytorch_cpuinfo
-      URL ${DEP_URL_pytorch_cpuinfo}
-      URL_HASH SHA1=${DEP_SHA1_pytorch_cpuinfo}
-      EXCLUDE_FROM_ALL
-      PATCH_COMMAND
-        ${Patch_EXECUTABLE} -p1 < ${PROJECT_SOURCE_DIR}/patches/cpuinfo/patch_cpuinfo_h_for_arm64ec.patch &&
-        # https://github.com/pytorch/cpuinfo/pull/324
-        ${Patch_EXECUTABLE} -p1 < ${PROJECT_SOURCE_DIR}/patches/cpuinfo/patch_vcpkg_arm64ec_support.patch
-      FIND_PACKAGE_ARGS NAMES cpuinfo
-    )
-  elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux")
-    message(STATUS "Applying sysfs fallback patch for cpuinfo on Linux")
-    onnxruntime_fetchcontent_declare(
-      pytorch_cpuinfo
-      URL ${DEP_URL_pytorch_cpuinfo}
-      URL_HASH SHA1=${DEP_SHA1_pytorch_cpuinfo}
-      EXCLUDE_FROM_ALL
-      PATCH_COMMAND
-        # https://github.com/microsoft/onnxruntime/issues/10038
-        ${Patch_EXECUTABLE} -p1 < ${PROJECT_SOURCE_DIR}/patches/cpuinfo/fix_missing_sysfs_fallback.patch
-      FIND_PACKAGE_ARGS NAMES cpuinfo
-    )
+  if(onnxruntime_USE_VCPKG AND NOT APPLE)
+    find_package(cpuinfo CONFIG REQUIRED)
   else()
-    onnxruntime_fetchcontent_declare(
-      pytorch_cpuinfo
-      URL ${DEP_URL_pytorch_cpuinfo}
-      URL_HASH SHA1=${DEP_SHA1_pytorch_cpuinfo}
-      EXCLUDE_FROM_ALL
-      FIND_PACKAGE_ARGS NAMES cpuinfo
-    )
+    if (onnxruntime_target_platform STREQUAL "ARM64EC" OR onnxruntime_target_platform STREQUAL "ARM64")
+      message(STATUS "Applying patches for Windows ARM64/ARM64EC in cpuinfo")
+      onnxruntime_fetchcontent_declare(
+        pytorch_cpuinfo
+        URL ${DEP_URL_pytorch_cpuinfo}
+        URL_HASH SHA1=${DEP_SHA1_pytorch_cpuinfo}
+        EXCLUDE_FROM_ALL
+        PATCH_COMMAND
+          ${Patch_EXECUTABLE} -p1 < ${PROJECT_SOURCE_DIR}/patches/cpuinfo/patch_cpuinfo_h_for_arm64ec.patch &&
+          # https://github.com/pytorch/cpuinfo/pull/324
+          ${Patch_EXECUTABLE} -p1 < ${PROJECT_SOURCE_DIR}/patches/cpuinfo/patch_vcpkg_arm64ec_support.patch &&
+          # https://github.com/pytorch/cpuinfo/pull/400
+          ${Patch_EXECUTABLE} --binary --ignore-whitespace -p1 <
+          ${PROJECT_SOURCE_DIR}/patches/cpuinfo/enable_deinit_refcounting.patch
+      )
+    elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+      message(STATUS "Applying sysfs fallback patch for cpuinfo on Linux")
+      onnxruntime_fetchcontent_declare(
+        pytorch_cpuinfo
+        URL ${DEP_URL_pytorch_cpuinfo}
+        URL_HASH SHA1=${DEP_SHA1_pytorch_cpuinfo}
+        EXCLUDE_FROM_ALL
+        PATCH_COMMAND
+          # https://github.com/microsoft/onnxruntime/issues/10038
+          ${Patch_EXECUTABLE} -p1 < ${PROJECT_SOURCE_DIR}/patches/cpuinfo/fix_missing_sysfs_fallback.patch &&
+          # https://github.com/pytorch/cpuinfo/pull/400
+          ${Patch_EXECUTABLE} --binary --ignore-whitespace -p1 <
+          ${PROJECT_SOURCE_DIR}/patches/cpuinfo/enable_deinit_refcounting.patch
+      )
+    else()
+      onnxruntime_fetchcontent_declare(
+        pytorch_cpuinfo
+        URL ${DEP_URL_pytorch_cpuinfo}
+        URL_HASH SHA1=${DEP_SHA1_pytorch_cpuinfo}
+        EXCLUDE_FROM_ALL
+        PATCH_COMMAND
+          # https://github.com/pytorch/cpuinfo/pull/400
+          ${Patch_EXECUTABLE} --binary --ignore-whitespace -p1 <
+          ${PROJECT_SOURCE_DIR}/patches/cpuinfo/enable_deinit_refcounting.patch
+      )
+    endif()
+    onnxruntime_fetchcontent_makeavailable(pytorch_cpuinfo)
   endif()
-  set(ONNXRUNTIME_CPUINFO_PROJ pytorch_cpuinfo)
-  onnxruntime_fetchcontent_makeavailable(${ONNXRUNTIME_CPUINFO_PROJ})
   if(TARGET cpuinfo::cpuinfo AND NOT TARGET cpuinfo)
     message(STATUS "Aliasing cpuinfo::cpuinfo to cpuinfo")
     add_library(cpuinfo ALIAS cpuinfo::cpuinfo)

cmake/external/xnnpack.cmake

@@ -84,9 +84,9 @@ endif()
 
 
 onnxruntime_fetchcontent_declare(googlexnnpack URL ${DEP_URL_googlexnnpack} URL_HASH SHA1=${DEP_SHA1_googlexnnpack}
-                     PATCH_COMMAND ${Patch_EXECUTABLE} --binary --ignore-whitespace -p1 < ${PROJECT_SOURCE_DIR}/patches/xnnpack/AddEmscriptenAndIosSupport.patch
+                     PATCH_COMMAND ${Patch_EXECUTABLE} --binary --ignore-whitespace -p1 < ${PROJECT_SOURCE_DIR}/patches/xnnpack/AddEmscriptenAndIosSupport.patch &&
+                                   ${Patch_EXECUTABLE} --ignore-whitespace -p1 < ${PROJECT_SOURCE_DIR}/patches/xnnpack/release_cpuinfo_after_hardware_config.patch
 		     EXCLUDE_FROM_ALL
-		     FIND_PACKAGE_ARGS NAMES xnnpack
                     )
 onnxruntime_fetchcontent_makeavailable(googlexnnpack)
 set(XNNPACK_DIR ${googlexnnpack_SOURCE_DIR})

cmake/onnxruntime_cuda_cccl.cmake

@@ -0,0 +1,95 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+# CCCL (libcu++/CUB/Thrust) header handling shared by the in-tree CUDA provider
+# (onnxruntime_providers_cuda.cmake) and the CUDA plugin EP (onnxruntime_providers_cuda_plugin.cmake).
+# Both compile host C++ translation units that include CUTLASS headers, which pull in <cuda/std/...>,
+# so both need the same include path handling.
+
+include_guard(GLOBAL)
+
+# Work around a CUDA 13.3 cudafe++ (EDG front-end) regression that mis-parses CCCL's
+# global-qualified partial specializations, e.g. in <cub/device/device_transform.cuh>:
+#   template <typename T>
+#   struct ::cuda::proclaims_copyable_arguments<...> : ::cuda::std::true_type {};
+# nvcc fails with "global qualification of class name is invalid before ':' token".
+# The fix is to write the specialization with the namespace reopened instead of using a
+# global-qualified name. We cannot edit the (often read-only) toolkit headers, so generate
+# corrected copies of the affected headers into the build tree and place that directory
+# ahead of the toolkit cccl include path. This is a no-op on toolkits whose headers do not
+# contain the offending pattern (e.g. once NVIDIA fixes it), so it is safe to keep enabled.
+function(ort_cuda133_patch_cccl_header src dst)
+  if (NOT EXISTS "${src}")
+    return()
+  endif()
+  file(READ "${src}" _content)
+  set(_orig "${_content}")
+  # <cub/device/device_transform.cuh>
+  string(REPLACE
+    "template <typename T>\nstruct ::cuda::proclaims_copyable_arguments<CUB_NS_QUALIFIER::detail::__return_constant<T>> : ::cuda::std::true_type\n{};"
+    "_CCCL_BEGIN_NAMESPACE_CUDA\ntemplate <typename T>\nstruct proclaims_copyable_arguments<CUB_NS_QUALIFIER::detail::__return_constant<T>> : ::cuda::std::true_type\n{};\n_CCCL_END_NAMESPACE_CUDA"
+    _content "${_content}")
+  # <cub/device/dispatch/tuning/tuning_transform.cuh>
+  string(REPLACE
+    "template <>\nstruct ::cuda::proclaims_copyable_arguments<CUB_NS_QUALIFIER::detail::transform::always_true_predicate>\n    : ::cuda::std::true_type\n{};"
+    "_CCCL_BEGIN_NAMESPACE_CUDA\ntemplate <>\nstruct proclaims_copyable_arguments<CUB_NS_QUALIFIER::detail::transform::always_true_predicate>\n    : ::cuda::std::true_type\n{};\n_CCCL_END_NAMESPACE_CUDA"
+    _content "${_content}")
+  if (NOT _content STREQUAL _orig)
+    get_filename_component(_dst_dir "${dst}" DIRECTORY)
+    file(MAKE_DIRECTORY "${_dst_dir}")
+    file(WRITE "${dst}" "${_content}")
+  elseif (EXISTS "${dst}")
+    # The toolkit header no longer matches the offending pattern (e.g. after a CUDA
+    # upgrade in an existing build tree). Remove any previously generated copy so a
+    # stale patched header does not keep shadowing the toolkit header.
+    file(REMOVE "${dst}")
+  endif()
+endfunction()
+
+# Give ${target} everything it needs to compile against the CCCL headers of a CUDA 13 toolkit:
+#
+#  * Handle the CUDA 13.0 CCCL header directory move: libcu++, CUB and Thrust moved from
+#    <toolkit>/include to <toolkit>/include/cccl, so <cuda/std/utility> - reached from the CUTLASS
+#    headers that host .cc files include - is no longer on the default include path of the host
+#    compiler. nvcc adds it by itself, so this only matters for targets that compile host C++.
+#  * On CUDA 13.3, generate the patched CCCL headers described above into ${CMAKE_BINARY_DIR}
+#    (and remove stale ones), then put that directory first on the include path.
+#
+# Note that this generates files in the build tree as a side effect, not just include flags.
+# It must be called for every target that compiles CUDA or CUTLASS-including host sources;
+# targets that inherit $<TARGET_PROPERTY:...,INCLUDE_DIRECTORIES> from such a target are
+# covered by the parent call, but only because that generator expression is evaluated after
+# configuration - see the call in onnxruntime_providers_cuda_plugin.cmake.
+function(ort_configure_cuda_cccl target)
+  if (CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 13.0)
+    return()
+  endif()
+
+  foreach(inc_dir ${CUDAToolkit_INCLUDE_DIRS})
+    if (EXISTS "${inc_dir}/cccl")
+      # The UNIX guard is not a statement about MSVC being unaffected: the cudafe++ regression
+      # is simply untested on Windows, where no CUDA 13.3 build has been run. If a Windows
+      # CUDA 13.3 build hits the same "global qualification of class name is invalid" error,
+      # dropping UNIX from this condition is expected to be all that is needed.
+      if (UNIX AND CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 13.3 AND CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 13.4)
+        # Generate cudafe++-parseable copies of the CCCL headers that contain global-qualified
+        # partial specializations (see ort_cuda133_patch_cccl_header above) and put the fixed
+        # directory ahead of the toolkit cccl include so the corrected headers win.
+        set(_ort_cccl_fix_dir "${CMAKE_BINARY_DIR}/cccl_cuda13_fix")
+        ort_cuda133_patch_cccl_header(
+          "${inc_dir}/cccl/cub/device/device_transform.cuh"
+          "${_ort_cccl_fix_dir}/cub/device/device_transform.cuh")
+        ort_cuda133_patch_cccl_header(
+          "${inc_dir}/cccl/cub/device/dispatch/tuning/tuning_transform.cuh"
+          "${_ort_cccl_fix_dir}/cub/device/dispatch/tuning/tuning_transform.cuh")
+        if (EXISTS "${_ort_cccl_fix_dir}/cub/device/device_transform.cuh" OR
+            EXISTS "${_ort_cccl_fix_dir}/cub/device/dispatch/tuning/tuning_transform.cuh")
+          target_include_directories(${target} BEFORE PRIVATE "${_ort_cccl_fix_dir}")
+        endif()
+      endif()
+
+      # Add the cccl subdirectory to the include path so <cuda/std/utility> can be found
+      target_include_directories(${target} PRIVATE "${inc_dir}/cccl")
+    endif()
+  endforeach()
+endfunction()

cmake/onnxruntime_providers_cuda.cmake

@@ -66,6 +66,7 @@
   )
 
   include(onnxruntime_cuda_source_filters.cmake)
+  include(onnxruntime_cuda_cccl.cmake)
   onnxruntime_filter_cuda_cu_sources(onnxruntime_cuda_contrib_ops_cu_srcs)
 
   if (NOT onnxruntime_USE_TRT_FUSED_ATTENTION)
@@ -217,44 +218,6 @@
     target_compile_definitions(onnxruntime_providers_cuda PRIVATE FILE_NAME=\"onnxruntime_providers_cuda.dll\")
   endif()
 
-  # Work around a CUDA 13.3 cudafe++ (EDG front-end) regression that mis-parses CCCL's
-  # global-qualified partial specializations, e.g. in <cub/device/device_transform.cuh>:
-  #   template <typename T>
-  #   struct ::cuda::proclaims_copyable_arguments<...> : ::cuda::std::true_type {};
-  # nvcc fails with "global qualification of class name is invalid before ':' token".
-  # The fix is to write the specialization with the namespace reopened instead of using a
-  # global-qualified name. We cannot edit the (often read-only) toolkit headers, so generate
-  # corrected copies of the affected headers into the build tree and place that directory
-  # ahead of the toolkit cccl include path. This is a no-op on toolkits whose headers do not
-  # contain the offending pattern (e.g. once NVIDIA fixes it), so it is safe to keep enabled.
-  function(ort_cuda133_patch_cccl_header src dst)
-    if (NOT EXISTS "${src}")
-      return()
-    endif()
-    file(READ "${src}" _content)
-    set(_orig "${_content}")
-    # <cub/device/device_transform.cuh>
-    string(REPLACE
-      "template <typename T>\nstruct ::cuda::proclaims_copyable_arguments<CUB_NS_QUALIFIER::detail::__return_constant<T>> : ::cuda::std::true_type\n{};"
-      "_CCCL_BEGIN_NAMESPACE_CUDA\ntemplate <typename T>\nstruct proclaims_copyable_arguments<CUB_NS_QUALIFIER::detail::__return_constant<T>> : ::cuda::std::true_type\n{};\n_CCCL_END_NAMESPACE_CUDA"
-      _content "${_content}")
-    # <cub/device/dispatch/tuning/tuning_transform.cuh>
-    string(REPLACE
-      "template <>\nstruct ::cuda::proclaims_copyable_arguments<CUB_NS_QUALIFIER::detail::transform::always_true_predicate>\n    : ::cuda::std::true_type\n{};"
-      "_CCCL_BEGIN_NAMESPACE_CUDA\ntemplate <>\nstruct proclaims_copyable_arguments<CUB_NS_QUALIFIER::detail::transform::always_true_predicate>\n    : ::cuda::std::true_type\n{};\n_CCCL_END_NAMESPACE_CUDA"
-      _content "${_content}")
-    if (NOT _content STREQUAL _orig)
-      get_filename_component(_dst_dir "${dst}" DIRECTORY)
-      file(MAKE_DIRECTORY "${_dst_dir}")
-      file(WRITE "${dst}" "${_content}")
-    elseif (EXISTS "${dst}")
-      # The toolkit header no longer matches the offending pattern (e.g. after a CUDA
-      # upgrade in an existing build tree). Remove any previously generated copy so a
-      # stale patched header does not keep shadowing the toolkit header.
-      file(REMOVE "${dst}")
-    endif()
-  endfunction()
-
   # config_cuda_provider_shared_module can be used to config onnxruntime_providers_cuda_obj, onnxruntime_providers_cuda & onnxruntime_providers_cuda_ut.
   # This function guarantees that all 3 targets have the same configurations.
   function(config_cuda_provider_shared_module target)
@@ -422,32 +385,7 @@
     target_link_libraries(${target} PRIVATE Eigen3::Eigen)
     target_include_directories(${target} PRIVATE ${ONNXRUNTIME_ROOT} ${CMAKE_CURRENT_BINARY_DIR} PUBLIC ${CUDAToolkit_INCLUDE_DIRS})
 
-    # Handle CUDA 13.0 CCCL header directory move
-    if (CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 13.0)
-      foreach(inc_dir ${CUDAToolkit_INCLUDE_DIRS})
-        if (EXISTS "${inc_dir}/cccl")
-          if (UNIX AND CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 13.3 AND CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 13.4)
-            # Generate cudafe++-parseable copies of the CCCL headers that contain global-qualified
-            # partial specializations (see ort_cuda133_patch_cccl_header above) and put the fixed
-            # directory ahead of the toolkit cccl include so the corrected headers win.
-            set(_ort_cccl_fix_dir "${CMAKE_CURRENT_BINARY_DIR}/cccl_cuda13_fix")
-            ort_cuda133_patch_cccl_header(
-              "${inc_dir}/cccl/cub/device/device_transform.cuh"
-              "${_ort_cccl_fix_dir}/cub/device/device_transform.cuh")
-            ort_cuda133_patch_cccl_header(
-              "${inc_dir}/cccl/cub/device/dispatch/tuning/tuning_transform.cuh"
-              "${_ort_cccl_fix_dir}/cub/device/dispatch/tuning/tuning_transform.cuh")
-            if (EXISTS "${_ort_cccl_fix_dir}/cub/device/device_transform.cuh" OR
-                EXISTS "${_ort_cccl_fix_dir}/cub/device/dispatch/tuning/tuning_transform.cuh")
-              target_include_directories(${target} BEFORE PRIVATE "${_ort_cccl_fix_dir}")
-            endif()
-          endif()
-
-          # Add the cccl subdirectory to the include path so <cuda/std/utility> can be found
-          target_include_directories(${target} PRIVATE "${inc_dir}/cccl")
-        endif()
-      endforeach()
-    endif()
+    ort_configure_cuda_cccl(${target})
 
     # ${CMAKE_CURRENT_BINARY_DIR} is so that #include "onnxruntime_config.h" inside tensor_shape.h is found
     set_target_properties(${target} PROPERTIES LINKER_LANGUAGE CUDA)

cmake/onnxruntime_providers_cuda_plugin.cmake

@@ -111,6 +111,7 @@ list(FILTER CUDA_PLUGIN_EP_CU_SRCS EXCLUDE REGEX ".*/contrib_ops/cuda/transforme
 
 # Apply shared CUDA .cu source filtering (flash attention quick build, MoE GEMM FP4/FP8).
 include(onnxruntime_cuda_source_filters.cmake)
+include(onnxruntime_cuda_cccl.cmake)
 onnxruntime_filter_cuda_cu_sources(CUDA_PLUGIN_EP_CU_SRCS)
 onnxruntime_extract_sm_specific_cuda_sources(CUDA_PLUGIN_EP_CU_SRCS
   SM90_SOURCES _cuda_plugin_sm90_tma_srcs
@@ -447,6 +448,18 @@ target_include_directories(onnxruntime_providers_cuda_plugin PRIVATE
     ${cutlass_SOURCE_DIR}/tools/util/include
 )
 
+# The host .cc files globbed into this target (contrib_ops/cuda/llm/*.cc and friends) include
+# CUTLASS headers, which reach <cuda/std/...>. In the non-plugin build the same files are part
+# of onnxruntime_providers_cuda, which gets this from config_cuda_provider_shared_module.
+#
+# The SM-specific OBJECT libraries created above are covered by this call even though they
+# already exist: onnxruntime_add_cuda_plugin_object_library gives them this target's includes
+# as $<TARGET_PROPERTY:onnxruntime_providers_cuda_plugin,INCLUDE_DIRECTORIES>, which is
+# evaluated after configuration and so picks up whatever is added here - order included, so
+# the CUDA 13.3 patched-header directory keeps shadowing the toolkit CCCL headers for their
+# .cu sources. Keep that indirection in mind before making the inheritance eager.
+ort_configure_cuda_cccl(onnxruntime_providers_cuda_plugin)
+
 onnxruntime_add_include_to_target(
     onnxruntime_providers_cuda_plugin
     onnxruntime_common