From 9260034a22669ee07a13b2661276caf976338348 Mon Sep 17 00:00:00 2001 From: shenyunheng3 Date: Thu, 12 Nov 2026 19:00:38 +0800 Subject: [PATCH 1/2] bugfix: publish hccl completion after stream retirement. Defer failed slice publication until stream synchronization and the existing retry loop finish. Publish every slice exactly once so terminal counters cannot authorize early buffer reuse. Retain borrowed buffers while stream synchronization still fails; report abort errors and preserve explicit transfer failure after retirement. --- .../hccl_transport/hccl_transport.cpp | 29 +++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/mooncake-transfer-engine/src/transport/ascend_transport/hccl_transport/hccl_transport.cpp b/mooncake-transfer-engine/src/transport/ascend_transport/hccl_transport/hccl_transport.cpp index 9bb7db212..71d392470 100644 --- a/mooncake-transfer-engine/src/transport/ascend_transport/hccl_transport/hccl_transport.cpp +++ b/mooncake-transfer-engine/src/transport/ascend_transport/hccl_transport/hccl_transport.cpp @@ -133,7 +133,10 @@ void HcclTransport::initiatorLoop(int deviceLogicId, int selfIdx) { << ", source_addr: " << slice->source_addr << ", dest_addr: " << slice->hccl.dest_addr << ", ret: " << ret; - slice->markFailed(); + // Keep the error private until the stream and all retries + // have retired. Publishing FAILED here lets callers free + // buffers that a sibling operation still accesses. + slice->status = Slice::FAILED; } } @@ -146,7 +149,7 @@ void HcclTransport::initiatorLoop(int deviceLogicId, int selfIdx) { << ", remote devicePhyId: " << remote_rank_info_.devicePhyId << ", ret: " << ret; for (auto slice : slice_list) { - slice->markFailed(); + slice->status = Slice::FAILED; } } auto addOpfence = std::chrono::high_resolution_clock::now(); @@ -159,9 +162,21 @@ void HcclTransport::initiatorLoop(int deviceLogicId, int selfIdx) { << local_rank_info_.devicePhyId << ", remote devicePhyId: " << remote_rank_info_.devicePhyId << ", ret: " << ret; - aclrtStreamAbort(stream); + const int abort_ret = aclrtStreamAbort(stream); + if (abort_ret) { + LOG(ERROR) << "aclrtStreamAbort failed, ret: " << abort_ret; + } // Synchronize to drop ACL_ERROR_RT_STREAM_ABORT error - aclrtSynchronizeStream(stream); + int drain_ret = aclrtSynchronizeStream(stream); + while (drain_ret != ACL_SUCCESS) { + // An error result alone cannot release borrowed buffers. + // Keep the batch pending until the stream actually drains. + LOG_EVERY_N_SEC(ERROR, 1) + << "Waiting for failed HCCL stream retirement, ret: " + << drain_ret; + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + drain_ret = aclrtSynchronizeStream(stream); + } if (loop < transfer_max_retry_cnt) { LOG(INFO) << "Hccl transport failed. Retry ..."; @@ -180,7 +195,7 @@ void HcclTransport::initiatorLoop(int deviceLogicId, int selfIdx) { LOG(ERROR) << "Hccl transport failed after " << transfer_max_retry_cnt << " retries."; for (auto slice : slice_list) { - slice->markFailed(); + slice->status = Slice::FAILED; } } @@ -221,7 +236,9 @@ void HcclTransport::initiatorLoop(int deviceLogicId, int selfIdx) { } for (auto slice : slice_list) { - if (slice->status != Slice::SliceStatus::FAILED) { + if (slice->status == Slice::SliceStatus::FAILED) { + slice->markFailed(); + } else { slice->markSuccess(); } } From 50a9dd59df231254c976465dc840236b64b83d40 Mon Sep 17 00:00:00 2001 From: shenyunheng3 Date: Fri, 13 Nov 2026 13:57:10 +0800 Subject: [PATCH 2/2] bugfix: use glog time throttling for retirement. Use LOG_EVERY_T from glog0.6 for failed stream retirement diagnostics. LOG_EVERY_N_SEC belongs to another logging API and prevents the HCCL source from compiling. --- .../ascend_transport/hccl_transport/hccl_transport.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mooncake-transfer-engine/src/transport/ascend_transport/hccl_transport/hccl_transport.cpp b/mooncake-transfer-engine/src/transport/ascend_transport/hccl_transport/hccl_transport.cpp index 71d392470..b50cf488c 100644 --- a/mooncake-transfer-engine/src/transport/ascend_transport/hccl_transport/hccl_transport.cpp +++ b/mooncake-transfer-engine/src/transport/ascend_transport/hccl_transport/hccl_transport.cpp @@ -171,7 +171,7 @@ void HcclTransport::initiatorLoop(int deviceLogicId, int selfIdx) { while (drain_ret != ACL_SUCCESS) { // An error result alone cannot release borrowed buffers. // Keep the batch pending until the stream actually drains. - LOG_EVERY_N_SEC(ERROR, 1) + LOG_EVERY_T(ERROR, 1) << "Waiting for failed HCCL stream retirement, ret: " << drain_ret; std::this_thread::sleep_for(std::chrono::milliseconds(1));