Skip to content

CVS-192139 Raise Legacy EP support for com.microsoft.GatherBlockQuantized #1242

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Bhumika-Meena-101 wants to merge 1 commit into
base: ovep-develop
Choose a base branch
from
Open

CVS-192139 Raise Legacy EP support for com.microsoft.GatherBlockQuantized #1242

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions onnxruntime/core/providers/openvino/ov_versions/capability.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,14 +41,18 @@
npu_qdq_optimizer_enabled = true; // see data_ops.cc ~615 where we check for int16 types for gpu, this may change to a better approach later
}

#if OPENVINO_VERSION_MAJOR == 2026 && OPENVINO_VERSION_MINOR == 2
#if OPENVINO_VERSION_MAJOR == 2026 && OPENVINO_VERSION_MINOR == 4
data_ops_ = std::make_unique<DataOps>(graph_viewer_, V_2026_4, device_type_, npu_qdq_optimizer_enabled);
#elif OPENVINO_VERSION_MAJOR == 2026 && OPENVINO_VERSION_MINOR == 3
data_ops_ = std::make_unique<DataOps>(graph_viewer_, V_2026_3, device_type_, npu_qdq_optimizer_enabled);
#elif OPENVINO_VERSION_MAJOR == 2026 && OPENVINO_VERSION_MINOR == 2
data_ops_ = std::make_unique<DataOps>(graph_viewer_, V_2026_2, device_type_, npu_qdq_optimizer_enabled);
#elif OPENVINO_VERSION_MAJOR == 2026 && OPENVINO_VERSION_MINOR == 1
data_ops_ = std::make_unique<DataOps>(graph_viewer_, V_2026_1, device_type_, npu_qdq_optimizer_enabled);
#elif OPENVINO_VERSION_MAJOR == 2026 && OPENVINO_VERSION_MINOR == 0
data_ops_ = std::make_unique<DataOps>(graph_viewer_, V_2026_0, device_type_, npu_qdq_optimizer_enabled);
#else
data_ops_ = std::make_unique<DataOps>(graph_viewer_, V_2026_2, device_type_, npu_qdq_optimizer_enabled);
data_ops_ = std::make_unique<DataOps>(graph_viewer_, V_2026_4, device_type_, npu_qdq_optimizer_enabled);

Check warning on line 55 in onnxruntime/core/providers/openvino/ov_versions/capability.cc

View workflow job for this annotation

GitHub Actions / Optional Lint C++ 

[cpplint] reported by reviewdog 🐶
 Add #include <memory> for make_unique<>  [build/include_what_you_use] [4]

Raw Output:
onnxruntime/core/providers/openvino/ov_versions/capability.cc:55:  Add #include <memory> for make_unique<>  [build/include_what_you_use] [4]
#endif
}

Expand Down
105 changes: 100 additions & 5 deletions onnxruntime/core/providers/openvino/ov_versions/data_ops.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@
{"FusedGemm", V_2023_0, {"CPU", "GPU"}},
{"FusedMatMul", V_2025_0, {"CPU", "GPU"}},
{"Gather", V_2020_4, {"CPU", "GPU"}},
{"GatherBlockQuantized", V_2026_4, {"CPU", "GPU", "NPU"}},
{"GatherElements", V_2022_2, {"CPU", "GPU"}},
{"GatherND", V_2021_4, {"CPU", "GPU"}},
{"Gelu", V_2023_1, {"CPU", "GPU"}},
Expand Down Expand Up @@ -408,7 +409,7 @@

// populate unsupportedmode_t
{
UnsupportedOpMode obj = {{V_2024_1, V_2024_2, V_2024_3, V_2024_4, V_2024_5, V_2024_6, V_2025_0, V_2025_1, V_2025_2, V_2025_3, V_2025_4, V_2026_0, V_2026_1, V_2026_2},
UnsupportedOpMode obj = {{V_2024_1, V_2024_2, V_2024_3, V_2024_4, V_2024_5, V_2024_6, V_2025_0, V_2025_1, V_2025_2, V_2025_3, V_2025_4, V_2026_0, V_2026_1, V_2026_2, V_2026_3, V_2026_4},
[this](const Node* node, const InitializedTensorSet&) {
// If the Input of ReduceMax op is UINT8, it is rejected (Due to output mismatch)
for (size_t i = 0; i < node->InputDefs().size(); i++) {
Expand All @@ -425,7 +426,7 @@
{
UnsupportedOpMode obj = {{V_2023_1, V_2023_2, V_2023_3, V_2024_0, V_2024_1, V_2024_2,
V_2024_3, V_2024_4, V_2024_5, V_2024_6, V_2025_0, V_2025_1,
V_2025_2, V_2025_3, V_2025_4, V_2026_0, V_2026_1, V_2026_2},
V_2025_2, V_2025_3, V_2025_4, V_2026_0, V_2026_1, V_2026_2, V_2026_3, V_2026_4},
[this](const Node* node, const InitializedTensorSet&) {
const auto& input_args = node->InputDefs();
const auto& input_arg = (input_args.size() > 1) ? input_args[1] : input_args[0];
Expand All @@ -445,7 +446,7 @@
{
UnsupportedOpMode obj = {{V_2023_1, V_2023_2, V_2023_3, V_2024_0, V_2024_1, V_2024_2,
V_2024_3, V_2024_4, V_2024_5, V_2024_6, V_2025_0, V_2025_1,
V_2025_2, V_2025_3, V_2025_4, V_2026_0, V_2026_1, V_2026_2},
V_2025_2, V_2025_3, V_2025_4, V_2026_0, V_2026_1, V_2026_2, V_2026_3, V_2026_4},
[this](const Node* node, const InitializedTensorSet&) {
// If the operator is unsqueeze
// If axes is an input, then we cannot produce a static graph.
Expand All @@ -461,7 +462,7 @@
}
{
UnsupportedOpMode obj = {{V_2023_1, V_2023_2, V_2023_3, V_2024_0, V_2024_1, V_2024_2, V_2024_3, V_2024_4, V_2024_5,
V_2024_6, V_2025_0, V_2025_1, V_2025_2, V_2025_3, V_2025_4, V_2026_0, V_2026_1, V_2026_2},
V_2024_6, V_2025_0, V_2025_1, V_2025_2, V_2025_3, V_2025_4, V_2026_0, V_2026_1, V_2026_2, V_2026_3, V_2026_4},
[this](const Node* node, const InitializedTensorSet&) {
// check for attributes
auto& upsample_attr = node->GetAttributes();
Expand Down Expand Up @@ -492,7 +493,7 @@
{
UnsupportedOpMode obj = {{V_2023_1, V_2023_2, V_2023_3, V_2024_0, V_2024_1, V_2024_2,
V_2024_3, V_2024_4, V_2024_5, V_2024_6, V_2025_0, V_2025_1,
V_2025_2, V_2025_3, V_2025_4, V_2026_0, V_2026_1, V_2026_2},
V_2025_2, V_2025_3, V_2025_4, V_2026_0, V_2026_1, V_2026_2, V_2026_3, V_2026_4},
[this](const Node* node, const InitializedTensorSet&) {
auto& attributes = node->GetAttributes();
if (attributes.count("coordinate_transformation_mode") > 0) {
Expand Down Expand Up @@ -724,6 +725,87 @@
return true;
}

bool DataOps::gather_block_quantized_unsupported(const Node* node) {
const auto& attributes = node->GetAttributes();
// Defaults mirror the contrib op schema (see contrib_defs.cc).
auto attr_or = [&attributes](const std::string& name, int64_t default_value) -> int64_t {
return attributes.count(name) > 0 ? attributes.at(name).i() : default_value;
};
const int64_t bits = attr_or("bits", 4);
const int64_t block_size = attr_or("block_size", 128);
const int64_t gather_axis = attr_or("gather_axis", 0);
const int64_t quantize_axis = attr_or("quantize_axis", 1);

const auto& input_defs = node->InputDefs();
// data, indices and scales are required; zero_points is optional.
if (input_defs.size() < 3) return true;
for (size_t i = 0; i < 3; ++i) {
if (input_defs[i] == nullptr || !input_defs[i]->Exists()) return true;
}

// data must be a constant initializer.
if (!graph_viewer_.IsConstantInitializer(input_defs[0]->Name(), true)) return true;

const auto* data_shape = input_defs[0]->Shape();
// Shape must be static to validate the layout constraints below.
if (data_shape == nullptr) return true;
const int rank = data_shape->dim_size();
if (rank < 2) return true;
for (const auto& dim : data_shape->dim()) {
if (!utils::HasDimValue(dim)) return true;
}

// Normalize the negative spellings before comparing axes.
const int64_t norm_gather_axis = gather_axis < 0 ? gather_axis + rank : gather_axis;
const int64_t norm_quantize_axis = quantize_axis < 0 ? quantize_axis + rank : quantize_axis;
if (norm_gather_axis < 0 || norm_gather_axis >= rank) return true;
if (norm_quantize_axis < 0 || norm_quantize_axis >= rank) return true;
if (norm_gather_axis == norm_quantize_axis) return true;

// quantize_axis must be the last dimension.
if (norm_quantize_axis != rank - 1) return true;

const auto* data_type = input_defs[0]->TypeAsProto();
if (data_type == nullptr) return true;
const auto data_elem_type = data_type->tensor_type().elem_type();
if (data_elem_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT8) {
// uint8 data supports bits 2, 4 and 8, and only gathers along axis 0.
if (bits != 2 && bits != 4 && bits != 8) return true;
if (norm_gather_axis != 0) return true;
} else if (data_elem_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT4 ||
data_elem_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT4) {
if (bits != 4) return true;
} else {
return true;
}

// block_size must be a power of 2 and at least 16.
if (block_size < 16 || (block_size & (block_size - 1)) != 0) return true;

// The quantized dimension must be an exact multiple of block_size.
if (data_shape->dim(rank - 1).dim_value() % block_size != 0) return true;

// scales, and zero_points when present, must be constants of matching rank.
for (size_t i = 2; i < input_defs.size(); ++i) {
const auto* input_def = input_defs[i];
if (input_def == nullptr || !input_def->Exists()) continue; // omitted optional input
if (!graph_viewer_.IsConstantInitializer(input_def->Name(), true)) return true;
const auto* shape = input_def->Shape();
if (shape == nullptr || shape->dim_size() != rank) return true;
for (const auto& dim : shape->dim()) {
if (!utils::HasDimValue(dim)) return true;
}
}

// zero_points must have the same type as data.
if (input_defs.size() > 3 && input_defs[3] != nullptr && input_defs[3]->Exists()) {
const auto* zp_type = input_defs[3]->TypeAsProto();
if (zp_type == nullptr || zp_type->tensor_type().elem_type() != data_elem_type) return true;
}

return false;
}

bool DataOps::node_is_supported(const NodeIndex node_idx, bool& has_external_weights_) {
const auto& node = graph_viewer_.GetNode(node_idx);
const auto& optype = node->OpType();
Expand All @@ -743,6 +825,7 @@
3. Check Op is supported
3a. Check if Op is of known unsupported modes (edge cases). If yes return false right away.
3b. If above is not true, check if the op is available in nGraph.
4. Check contrib op configurations that the OpenVINO ONNX frontend cannot translate.
*/

// Check 0
Expand Down Expand Up @@ -862,6 +945,18 @@
return false;
}

// Check 4: contrib ops with per-configuration limitations. Kept separate from
// op_list_ above, which is keyed on op type alone and only consulted for kOnnxDomain.
if (domain == kMSDomain && optype == "GatherBlockQuantized" &&
gather_block_quantized_unsupported(node)) {
#ifndef NDEBUG
if (openvino_ep::backend_utils::IsDebugEnabled()) {
std::cout << "GatherBlockQuantized configuration is not supported" << std::endl;

Check warning on line 954 in onnxruntime/core/providers/openvino/ov_versions/data_ops.cc

View workflow job for this annotation

GitHub Actions / Optional Lint C++ 

[cpplint] reported by reviewdog 🐶
 Add #include <iostream> for cout  [build/include_what_you_use] [4]

Raw Output:
onnxruntime/core/providers/openvino/ov_versions/data_ops.cc:954:  Add #include <iostream> for cout  [build/include_what_you_use] [4]
}
#endif
return false;
}

return true;
}

Expand Down
5 changes: 4 additions & 1 deletion onnxruntime/core/providers/openvino/ov_versions/data_ops.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,9 @@ enum versionNum {
V_2025_4,
V_2026_0,
V_2026_1,
V_2026_2
V_2026_2,
V_2026_3,
V_2026_4
};

using VersionNum = enum versionNum;
Expand Down Expand Up @@ -82,6 +84,7 @@ class DataOps {
bool op_is_supported(std::string name, std::vector<SupportedOp>& list);
bool dimension_unsupported(const Node* node);
bool unsupported_op_mode(const Node* node, bool& has_external_weights_);
bool gather_block_quantized_unsupported(const Node* node);
bool type_is_supported(const NodeArg* node_arg, bool is_initializer);
bool node_is_supported(const NodeIndex node_idx, bool& has_external_weights_);

Expand Down
Loading