Repository navigation
feat(qy): QY ops backend with FA and longctx MoE/RoPE kernels (2026.11.12) #1038
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
Ceng23333
wants to merge
7
commits into
InfiniTensor:master
Choose a base branch
from
Ceng23333:feat/qy-backend-20261112
base: master
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from all commits
Commits
Show all changes
7 commits
Select commit
Hold shift + click to select a range
772130d
feat(qy): add QY ops backend with linked FA and paged KV flash
Ceng23333 97adba6
feat(qy): add longctx MoE/RoPE QY kernels for MiniCPM5
Ceng23333 7ee8485
style(qy): clang-format QY backend changes
Ceng23333 27b3487
fix(qy): instantiate torch Operator templates for WITH_QY
9c43cfa
fix(qy): clang-format QY kernels and gate kQy for non-QY CI
71e8516
fix(qy): include QY C10 specialization in TorchStreamGuard
67a6b3c
ci: retrigger unit jobs after runner proxy flake
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| #ifndef INFINI_OPS_MOE_ALIGN_KERNEL_H_ | ||
| #define INFINI_OPS_MOE_ALIGN_KERNEL_H_ | ||
|
|
||
| #include <cstddef> | ||
| #include <cstdint> | ||
|
|
||
| namespace infini::ops::qy { | ||
|
|
||
| // Workspace bytes for moe_align_launch: 2 * (num_experts + 1) * sizeof(int32). | ||
| inline size_t moe_align_workspace_bytes(size_t num_experts) { | ||
| return 2 * (num_experts + 1) * sizeof(int32_t); | ||
| } | ||
|
|
||
| // Host-free contiguous moe_align (Denglin-safe block/grid caps). | ||
| // topk_ids / expert_map: int32; sorted_token_ids / expert_ids / | ||
| // num_tokens_post_padded: int32. expert_map may be nullptr. | ||
| void moe_align_launch(void* sorted_token_ids, void* expert_ids, | ||
| void* num_tokens_post_padded, const void* topk_ids, | ||
| const void* expert_map, size_t numel, size_t num_experts, | ||
| size_t block_size, size_t sorted_capacity, | ||
| size_t expert_ids_capacity, bool pad_sorted_token_ids, | ||
| void* workspace, void* stream); | ||
|
|
||
| } // namespace infini::ops::qy | ||
|
|
||
| #endif // INFINI_OPS_MOE_ALIGN_KERNEL_H_ |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| #ifndef INFINI_OPS_MOE_SUM_KERNEL_H_ | ||
| #define INFINI_OPS_MOE_SUM_KERNEL_H_ | ||
|
|
||
| #include <cstddef> | ||
|
|
||
| namespace infini::ops::qy { | ||
|
|
||
| // Contiguous expert combine: out[t, h] = sum_k in[t, k, h] | ||
| // in is [tokens, topk, hidden], out is [tokens, hidden]. | ||
| // dtype_tag matches infini::rt::DataType (kFloat16 / kBFloat16 / kFloat32). | ||
| void moe_sum_launch(void* out, const void* in, size_t tokens, size_t topk, | ||
| size_t hidden, int dtype_tag, void* stream); | ||
|
|
||
| } // namespace infini::ops::qy | ||
|
|
||
| #endif // INFINI_OPS_MOE_SUM_KERNEL_H_ |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| #ifndef INFINI_OPS_MOE_TOPK_SIGMOID_KERNEL_H_ | ||
| #define INFINI_OPS_MOE_TOPK_SIGMOID_KERNEL_H_ | ||
|
|
||
| #include <cstddef> | ||
|
|
||
| namespace infini::ops::qy { | ||
|
|
||
| // Contiguous MoE top-k after sigmoid (no CUB). | ||
| // gating: [tokens, experts] (f16/bf16/f32) | ||
| // correction_bias: [experts] (f16/bf16/f32) or null; bias_dtype_tag ignored if | ||
| // null topk_weights: [tokens, topk] (f16/bf16/f32); topk_indices: [tokens, | ||
| // topk] (i32/i64) dtype tags match infini::rt::DataType. | ||
| void moe_topk_sigmoid_launch(void* topk_weights, void* topk_indices, | ||
| const void* gating, const void* correction_bias, | ||
| size_t tokens, size_t experts, size_t topk, | ||
| bool renormalize, int gating_dtype_tag, | ||
| int weight_dtype_tag, int index_dtype_tag, | ||
| int bias_dtype_tag, void* stream); | ||
|
|
||
| } // namespace infini::ops::qy | ||
|
|
||
| #endif // INFINI_OPS_MOE_TOPK_SIGMOID_KERNEL_H_ |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| #ifndef INFINI_OPS_MUL_SCALAR_KERNEL_H_ | ||
| #define INFINI_OPS_MUL_SCALAR_KERNEL_H_ | ||
|
|
||
| #include <cstddef> | ||
|
|
||
| namespace infini::ops::qy { | ||
|
|
||
| // Contiguous elementwise out[i] = in[i] * alpha. | ||
| // dtype_tag matches infini::rt::DataType (kFloat16 / kBFloat16 / kFloat32 / | ||
| // kFloat64). | ||
| void mul_scalar_launch(void* out, const void* in, size_t numel, int dtype_tag, | ||
| double alpha, void* stream); | ||
|
|
||
| } // namespace infini::ops::qy | ||
|
|
||
| #endif // INFINI_OPS_MUL_SCALAR_KERNEL_H_ |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| #ifndef INFINI_OPS_ROPE_KERNEL_H_ | ||
| #define INFINI_OPS_ROPE_KERNEL_H_ | ||
|
|
||
| #include <cstddef> | ||
| #include <cstdint> | ||
|
|
||
| namespace infini::ops::qy { | ||
|
|
||
| // Contiguous packed RoPE on [seqlen, nhead, head_dim] (nhead may be 1). | ||
| // pos_ids: int64 [seqlen]; sin/cos tables: [max_pos, head_dim/2], same dtype as | ||
| // x. algo_tag: 0 = GPT_J (interleaved), 1 = GPT_NEOX (half-split). dtype_tag | ||
| // matches infini::rt::DataType (kFloat16 / kBFloat16 / kFloat32). | ||
| void rope_launch(void* out, const void* in, const int64_t* pos_ids, | ||
| const void* sin_table, const void* cos_table, size_t seqlen, | ||
| size_t nhead, size_t head_dim, int algo_tag, int dtype_tag, | ||
| void* stream); | ||
|
|
||
| } // namespace infini::ops::qy | ||
|
|
||
| #endif // INFINI_OPS_ROPE_KERNEL_H_ |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| #ifndef INFINI_OPS_ROPE_PACK_KERNEL_H_ | ||
| #define INFINI_OPS_ROPE_PACK_KERNEL_H_ | ||
|
|
||
| #include <cstddef> | ||
| #include <cstdint> | ||
|
|
||
| namespace infini::ops::qy { | ||
|
|
||
| // Pack strided [seq, heads, dim] (elem_bytes 2 or 4) into contiguous row-major. | ||
| // src_stride{0,1,2} are in elements. dst is contiguous [seq, heads, dim]. | ||
| void rope_pack_launch(void* dst, const void* src, size_t seq, size_t heads, | ||
| size_t dim, int64_t src_stride0, int64_t src_stride1, | ||
| int64_t src_stride2, int elem_bytes, void* stream); | ||
|
|
||
| // Unpack contiguous [seq, heads, dim] into strided dst with the given strides. | ||
| void rope_unpack_launch(void* dst, const void* src, size_t seq, size_t heads, | ||
| size_t dim, int64_t dst_stride0, int64_t dst_stride1, | ||
| int64_t dst_stride2, int elem_bytes, void* stream); | ||
|
|
||
| } // namespace infini::ops::qy | ||
|
|
||
| #endif // INFINI_OPS_ROPE_PACK_KERNEL_H_ |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| #ifndef INFINI_OPS_SILU_AND_MUL_KERNEL_H_ | ||
| #define INFINI_OPS_SILU_AND_MUL_KERNEL_H_ | ||
|
|
||
| #include <cstddef> | ||
|
|
||
| namespace infini::ops::qy { | ||
|
|
||
| // Contiguous gated SiLU: out[r, c] = silu(in[r, c]) * in[r, c + hidden] | ||
| // in is [rows, 2*hidden], out is [rows, hidden]. | ||
| // dtype_tag matches infini::rt::DataType (kFloat16 / kBFloat16 / kFloat32). | ||
| void silu_and_mul_launch(void* out, const void* in, size_t rows, size_t hidden, | ||
| int dtype_tag, void* stream); | ||
|
|
||
| } // namespace infini::ops::qy | ||
|
|
||
| #endif // INFINI_OPS_SILU_AND_MUL_KERNEL_H_ |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
CMake 清空用户指定的 QY_ROOT。 -DQY_ROOT=... 被覆盖,非默认 SDK 路径配置失败