diff --git a/dkms.conf b/dkms.conf index d1c7440..f09b0f1 100644 --- a/dkms.conf +++ b/dkms.conf @@ -1,5 +1,5 @@ PACKAGE_NAME=aws-neuronx -PACKAGE_VERSION=2.29.0.0 +PACKAGE_VERSION=2.30.2.0 BUILT_MODULE_NAME[0]="neuron" MAKE[0]="make -C ${kernel_source_dir} M=${dkms_tree}/${PACKAGE_NAME}/${PACKAGE_VERSION}/build" CLEAN="make -C ${kernel_source_dir} M=${dkms_tree}/${PACKAGE_NAME}/${PACKAGE_VERSION}/build clean" diff --git a/neuron_cdev.c b/neuron_cdev.c index e64679e..dd6f776 100644 --- a/neuron_cdev.c +++ b/neuron_cdev.c @@ -779,7 +779,12 @@ static int ncdev_get_dmabuf_fd_v2(void *param) return 0; err_close_fd: +#if LINUX_VERSION_CODE > KERNEL_VERSION(5, 10, 192) close_fd(dmabuf_fd); +#else + __close_fd(current->files, dmabuf_fd); +#endif + return -EFAULT; } @@ -864,6 +869,8 @@ static int ncdev_mem_copy(struct neuron_device *nd, unsigned int cmd, void *para u64 src_offset; u64 dst_offset; u64 size; + u64 sequence_num = 0; + void *context = NULL; int ret; if (cmd == NEURON_IOCTL_MEM_COPY) { @@ -876,7 +883,16 @@ static int ncdev_mem_copy(struct neuron_device *nd, unsigned int cmd, void *para src_offset = arg.src_offset; dst_offset = arg.dst_offset; size = arg.size; - dst_mc = ncdev_mem_handle_to_mem_chunk(nd, arg.dst_mem_handle); + } else if (cmd == NEURON_IOCTL_MEM_COPY64_DEPRECATED) { + struct neuron_ioctl_mem_copy64_deprecated arg; + ret = neuron_copy_from_user(__func__, &arg, (struct neuron_ioctl_mem_copy64_deprecated *)param, sizeof(arg)); + if (ret) + return ret; + src_mem_handle = arg.src_mem_handle; + dst_mem_handle = arg.dst_mem_handle; + src_offset = arg.src_offset; + dst_offset = arg.dst_offset; + size = arg.size; } else if (cmd == NEURON_IOCTL_MEM_COPY64) { struct neuron_ioctl_mem_copy64 arg; ret = neuron_copy_from_user(__func__, &arg, (struct neuron_ioctl_mem_copy64 *)param, sizeof(arg)); @@ -887,6 +903,8 @@ static int ncdev_mem_copy(struct neuron_device *nd, unsigned int cmd, void *para src_offset = arg.src_offset; dst_offset = arg.dst_offset; size = arg.size; + sequence_num = arg.sequence_num; + context = arg.context; } else { return -EINVAL; } @@ -908,11 +926,21 @@ static int ncdev_mem_copy(struct neuron_device *nd, unsigned int cmd, void *para } ret = ndma_memcpy_mc(nd, src_mc, dst_mc, src_offset, dst_offset, size); if (ret) { - pr_err("dma memcpy failed\n"); return ret; } + if (sequence_num != 0) { + u32 nc_id = ndma_mc_pair_to_nc(src_mc, dst_mc); + int qid = ndhal->ndhal_ndmar.ndmar_get_h2t_def_qid(nc_id); + int submit_ret; + + // Temporary fake async copy: the copy is already done, but report its result through CQE in queue order. + submit_ret = ndma_zerocopy_submit_completed(nd, nc_id, qid, sequence_num, 0, context); + if (submit_ret) { + return submit_ret; + } + } trace_ioctl_mem_copy(nd, src_mc, dst_mc); - return 0; + return ret; } static int ncdev_mem_copy_async(struct neuron_device *nd, unsigned int cmd, void *param) @@ -1256,6 +1284,8 @@ static int ncdev_mem_buf_zerocopy64(struct neuron_device *nd, unsigned int cmd, u64 size; u32 bar4_wr_threshold; int h2t_qid; + u32 nc_id; + int qid; int ret; struct neuron_ioctl_mem_buf_copy64zc arg; bool use_bar4_wr; @@ -1292,6 +1322,13 @@ static int ncdev_mem_buf_zerocopy64(struct neuron_device *nd, unsigned int cmd, return -EFAULT; } + nc_id = ndma_mc_pair_to_nc(mc, mc); + qid = (h2t_qid == NEURON_DMA_H2T_DEFAULT_QID) ? ndhal->ndhal_ndmar.ndmar_get_h2t_def_qid(nc_id) : h2t_qid; + if (!ndmar_qid_valid(qid)) { + pr_err("nd%02d: invalid h2t queue index %d", nd->device_index, qid); + return -ENOENT; + } + // limit to internal threshold to prevent DoS attack bar4_wr_threshold = (arg.bar4_wr_threshold < BAR4_WR_THRESHOLD_MAX) ? arg.bar4_wr_threshold : BAR4_WR_THRESHOLD_MAX; use_bar4_wr = !narch_is_qemu() && @@ -1319,18 +1356,17 @@ static int ncdev_mem_buf_zerocopy64(struct neuron_device *nd, unsigned int cmd, if (unlikely(ret)) { ret = neuron_copy_from_user(__func__, nd->npdev.bar4 + cpy_offset, buffer, size); } + + // for async mode, BAR4 write is already done; enqueue a completed ctx to preserve CQE order. + // for sync mode, return the BAR4 write result directly + if (sequence_num != 0) { + ret = ndma_zerocopy_submit_completed(nd, nc_id, qid, sequence_num, ret, context); + } + return ret; } else { nrt_tensor_batch_op_t op; - - u32 nc_id = ndma_mc_pair_to_nc(mc, mc); - int qid = (h2t_qid == NEURON_DMA_H2T_DEFAULT_QID) ? ndhal->ndhal_ndmar.ndmar_get_h2t_def_qid(nc_id) : h2t_qid; dma_addr_t dev_base = ndma_mc_to_pa(mc); // the caller already does the range check for dev_base+offset - if (!ndmar_qid_valid(qid)) { - pr_err("nd%02d: invalid h2t queue index %d", nd->device_index, qid); - return -ENOENT; - } - if (!ndma_zerocopy_supported()) { pr_err_once("nd%02d: zero copy is not supported for architectures requiring DMA retry", nd->device_index); return -EINVAL; @@ -1343,9 +1379,8 @@ static int ncdev_mem_buf_zerocopy64(struct neuron_device *nd, unsigned int cmd, ret = ndma_zerocopy_submit(nd, nc_id, &op, 1, dev_base, qid, copy_to_mem_handle ? true : false, sequence_num, context); + return ret; } - - return ret; } static int ncdev_mem_buf_zerocopy64_batch(struct neuron_device *nd, void *param) @@ -2027,7 +2062,8 @@ static long ncdev_driver_info(unsigned int cmd, void *param) NEURON_DRIVER_FEATURE_ZEROCOPY | NEURON_DRIVER_FEATURE_PINNED_HOST_MEM | NEURON_DRIVER_FEATURE_ALLOC_WITH_PA; if (ndma_zerocopy_supported()) - driver_info.feature_flags1 |= NEURON_DRIVER_FEATURE_ASYNC_IO; + driver_info.feature_flags1 |= NEURON_DRIVER_FEATURE_ASYNC_RW | + NEURON_DRIVER_FEATURE_ASYNC_COPY; return copy_to_user(param, &driver_info, sizeof(driver_info)); } @@ -3342,6 +3378,34 @@ inline static long ncdev_misc_ioctl(struct file *filep, unsigned int cmd, unsign return -EINVAL; } +static int ncdev_get_fw_bars(struct neuron_device *nd, void *param) +{ + struct neuron_ioctl_get_bar_info arg; + int ret; + int i; + + ret = neuron_copy_from_user(__func__, &arg, (struct neuron_ioctl_get_bar_info *)param, sizeof(arg)); + if (ret) { + return ret; + } + + if (arg.query_type == 0 || arg.query_type > NEURON_MAX_BAR_QUERY_TYPES) { + return -EINVAL; + } + + if (!nd->bar_info[arg.query_type - 1].cached) { + return -EAGAIN; + } + + arg.count = nd->bar_info[arg.query_type - 1].count; + for (i = 0; i < arg.count; i++) { + arg.bar_types[i] = nd->bar_info[arg.query_type - 1].bars[i].bar_type; + arg.bars[i] = nd->bar_info[arg.query_type - 1].bars[i].bar_address; + } + + return copy_to_user(param, &arg, sizeof(arg)); +} + static long ncdev_ioctl(struct file *filep, unsigned int cmd, unsigned long param) { struct ncdev *ncd; @@ -3538,6 +3602,8 @@ static long ncdev_ioctl(struct file *filep, unsigned int cmd, unsigned long para return ncdev_available_perf_profiles(nd, (void*)param); } else if (cmd == NEURON_IOCTL_GET_ASYNC_H2T_DMA_COMPL_QUEUES) { return ncdev_get_async_h2d_dma_compl_queues(nd, (void*)param); + } else if (cmd == NEURON_IOCTL_GET_BAR_INFO) { + return ncdev_get_fw_bars(nd, (void*)param); } // B/W compatibility diff --git a/neuron_device.h b/neuron_device.h index b6a9190..8a89bf3 100644 --- a/neuron_device.h +++ b/neuron_device.h @@ -26,6 +26,7 @@ #include "neuron_sysfs_metrics.h" #include "neuron_log.h" #include "neuron_power.h" +#include "neuron_fw_io.h" #ifndef static_assert #define static_assert(expr, ...) @@ -42,6 +43,9 @@ // Global host memory buf size used for memset the device memory #define MEMSET_HOST_BUF_SIZE MAX_DMA_DESC_SIZE // guessed optimal DMA transfer and PCIe TLP size. +// PCI BAR addresses on switched fabrics +#define NEURON_MAX_BAR_QUERY_TYPES 2 + struct neuron_pci_device { phys_addr_t bar0_pa; void __iomem *bar0; @@ -67,6 +71,12 @@ struct neuron_hbm_scrub_ctx { struct mem_chunk *hostbuf_mc[NUM_DMA_ENG_PER_DEVICE]; }; +struct neuron_device_bar_info { + bool cached; + u8 count; + struct fw_io_bar_entry bars[NEURON_MAX_SWITCH_BARS]; +}; + struct neuron_device { struct pci_dev *pdev; int device_index; @@ -120,6 +130,8 @@ struct neuron_device { struct neuron_hbm_scrub_ctx hbm_scrub_ctx; + struct neuron_device_bar_info bar_info[NEURON_MAX_BAR_QUERY_TYPES]; + // volatile to prevent compiler optimizations since accessed by different threads // Indicates whether any performance profile with 7200 Mhz HBM is supported by this device volatile int supports_hbm_7200; diff --git a/neuron_dhal.h b/neuron_dhal.h index 71548b5..febfd03 100644 --- a/neuron_dhal.h +++ b/neuron_dhal.h @@ -78,6 +78,7 @@ struct ndhal_nc { struct ndhal_nq { u8 (*nnq_get_nqid) (struct neuron_device *nd, u8 nc_id, u8 index, u32 nq_type); void (*nnq_set_hwaddr) (struct neuron_device *nd, u8 nc_id, u8 index, u32 nq_type, u32 size, u64 queue_pa); + u8 (*nnq_get_nq_queue_count) (void); }; struct ndhal_mpset { @@ -106,6 +107,7 @@ struct ndhal_fw_io { int (*fw_io_read_csr_array) (void **addrs, u32 *values, u32 num_csrs, bool operational); int (*fw_io_execute_request) (struct fw_io_ctx *ctx, u8 command_id, const u8 *req, u32 req_size, u8 *resp, u32 resp_size); int (*fw_io_post_metric) (struct fw_io_ctx *ctx, u8 *data, u32 size); + void (*fw_io_cache_bar_info) (struct neuron_device *nd); }; struct ndhal_mmap { diff --git a/neuron_dma.c b/neuron_dma.c index fc768ac..ff878d9 100644 --- a/neuron_dma.c +++ b/neuron_dma.c @@ -949,7 +949,7 @@ enum ndma_zcdma_state { NDMA_UNPINNED, NDMA_PINNED_UNSUBMITTED, NDMA_SUBMITTED, - NDMA_COMPLETED, // not in dma context queue anymore + NDMA_COMPLETED, // completed transfer waiting to be harvested from ctx queue to CQ }; /* DMA context */ @@ -983,6 +983,8 @@ struct ndma_h2t_zcdma_context { // Async-only struct mm_struct *mm; // mm that owns the user buffers + s64 compl_ret; // result for in-order CQ completion of completed work. Only valid for async. + // used only for async BAR4 writes and temporary fake-async D2D copy }; static void ndma_pinned_mem_process_release(struct kref *kref); @@ -992,7 +994,7 @@ static void ndma_zc_release_ctx(struct ndma_h2t_zcdma_context *ctx, u64 *nr_pinn // do not free or set completion_ptr null. it is managed by completion_pool in ctx queue // do not free or set page_list null. it is managed by page_list_pool in ctx queue - if (ctx->state >= NDMA_PINNED_UNSUBMITTED) { + if (ctx->state >= NDMA_PINNED_UNSUBMITTED && ctx->nr_pages > 0) { /* Only unpin if we pinned it ourselves (not pre-pinned memory) */ if (!ctx->prepin_proc) { if (ctx->direction) { @@ -1017,6 +1019,7 @@ static void ndma_zc_release_ctx(struct ndma_h2t_zcdma_context *ctx, u64 *nr_pinn ctx->state = NDMA_INVALID; ctx->sequence_num = 0; ctx->context = NULL; + ctx->compl_ret = 0; } /* H2D DMA Completion Queue (CQ) */ @@ -1068,10 +1071,15 @@ void ndma_h2d_compl_queue_destroy(struct ndma_h2d_compl_queue *compl_queue) compl_queue->tail = 0; } +/* + * Emit the CQE for an accepted async io request. Caller must hold the ring + * lock that serializes compl_queue->tail. Submission failures return an error + * to userspace and must not call this function to emit a CQE. + */ static void ndma_h2d_compl_queue_put(struct ndma_h2d_compl_queue *compl_queue, - u64 sequence_num, - s64 compl_ret, - void *context) + u64 sequence_num, + s64 compl_ret, + void *context) { u32 head = 0; u32 tail = 0; @@ -1178,7 +1186,7 @@ static void ndma_ctx_queue_inc_tail(struct ndma_ctx_queue *queue) // Assume the ctx at old tail is already filled by caller // Tail advance may also initialize/advance the pinned+unsubmitted and unpinned pointers struct ndma_h2t_zcdma_context *ctx = &queue->entries[old_tail]; - if (ctx->state == NDMA_PINNED_UNSUBMITTED) { + if (ctx->state == NDMA_PINNED_UNSUBMITTED || ctx->state == NDMA_COMPLETED) { if (ndma_ctx_queue_pinned_unsubmitted_empty(queue)) { // The first pinned+unsubmitted pointer appears at old_tail queue->first_pinned_unsubmitted = old_tail; @@ -1862,6 +1870,53 @@ int ndma_zerocopy_submit(struct neuron_device *nd, return ret; } +int ndma_zerocopy_submit_completed(struct neuron_device *nd, u32 nc_id, int qid, + u64 sequence_num, s64 compl_ret, void *context) +{ + const int eng_id = ndhal->ndhal_ndmar.ndmar_get_h2t_eng_id(nd, nc_id); + struct ndma_eng *eng = &nd->ndma_engine[eng_id]; + struct ndma_ring *ring = &eng->queues[qid].ring_info; + struct ndma_ctx_queue *ctx_queue = &ring->dma_ctx_queue; + struct ndma_h2t_zcdma_context *ctx; + int ret; + + if (!ndmar_h2t_ring_is_owner(ring, nc_id)) { + pr_err("nd%02d: attempting to use qid %d that was not assigned to nc %d\n", + nd->device_index, qid, nc_id); + return -ENOENT; + } + + ret = ndma_h2d_create_cmpltn_thread(nd); + if (ret) { + return ret; + } + + mutex_lock(&ring->h2t_ring_lock); + if (ndma_ctx_queue_is_full(ctx_queue)) { + mutex_unlock(&ring->h2t_ring_lock); + return -EBUSY; + } + + ctx = ndma_ctx_queue_peek_tail(ctx_queue); + ctx->eng = eng; + ctx->ring = ring; + ctx->direction = true; + ctx->last = true; + ctx->nr_pages = 0; + ctx->nr_desc = 0; + ctx->state = NDMA_COMPLETED; + ctx->sequence_num = sequence_num; + ctx->context = context; + ctx->compl_ret = compl_ret; + ndma_ctx_queue_inc_tail(ctx_queue); + mutex_unlock(&ring->h2t_ring_lock); + + atomic64_or(BIT_ULL(ndhal->ndhal_ndmar.ndmar_ctx_queue_bit(eng_id, qid)), + &nd->dma_cmpltn_thread.nonempty_ctxq_bitmap); + wake_up(&nd->dma_cmpltn_thread.wait_queue); + return 0; +} + /* The completion flow for completion, remote pinning, and submission. Async IO only */ static int ndma_zerocopy_complete(struct neuron_device *nd, struct ndma_eng *eng, @@ -1897,7 +1952,13 @@ static int ndma_zerocopy_complete(struct neuron_device *nd, } struct ndma_h2t_zcdma_context *submitted_ctx = ndma_ctx_queue_pop_submitted(ctx_queue); - ret = ndma_memcpy_wait_for_completion(eng, ring, submitted_ctx->nr_desc + 1, submitted_ctx->completion_ptr, true, false); + if (submitted_ctx->state == NDMA_COMPLETED) { + // Dummy context: BAR4/fake-async copy completed before enqueue. + ret = submitted_ctx->compl_ret; + } else { + ret = ndma_memcpy_wait_for_completion(eng, ring, submitted_ctx->nr_desc + 1, + submitted_ctx->completion_ptr, true, false); + } if (ret) { err = ret; pr_err("async h2d dma completion failed for seq num %llu: %d\n", submitted_ctx->sequence_num, ret); @@ -1915,7 +1976,16 @@ static int ndma_zerocopy_complete(struct neuron_device *nd, while (true) { struct ndma_h2t_zcdma_context *pinned_unsubmitted_ctx = ndma_ctx_queue_peek_pinned_unsubmitted(ctx_queue); - if (!pinned_unsubmitted_ctx || !_ndma_zc_descs_available(eng, ring->qid, pinned_unsubmitted_ctx->nr_pages)) { + if (!pinned_unsubmitted_ctx) { + break; + } + if (pinned_unsubmitted_ctx->state == NDMA_COMPLETED) { + // Dummy context has no DMA descriptors; preserve CQ order only. + ndma_ctx_queue_inc_first_pinned_unsubmitted(ctx_queue); + did_work = true; + continue; + } + if (!_ndma_zc_descs_available(eng, ring->qid, pinned_unsubmitted_ctx->nr_pages)) { break; } diff --git a/neuron_dma.h b/neuron_dma.h index 787e8f8..d724bc1 100644 --- a/neuron_dma.h +++ b/neuron_dma.h @@ -264,6 +264,24 @@ int ndma_zerocopy_submit(struct neuron_device *nd, u64 sequence_num, void *context); +/** + * ndma_zerocopy_submit_completed() - Enqueue completed work for ordered async CQE. + * + * Used only for async BAR4 writes and temporary fake-async D2D copy, where the work + * has already completed before we enqueue a dummy context. + * + * @nd: Neuron device. + * @nc_id: NeuronCore id that owns the H2D queue. + * @qid: H2D queue id. + * @sequence_num: Async sequence number returned to userspace. + * @compl_ret: Completion result to report in the CQE. + * @context: Userspace completion context to copy into the CQE. + * + * Return: 0 if queued, negative errno otherwise. + */ +int ndma_zerocopy_submit_completed(struct neuron_device *nd, u32 nc_id, int qid, + u64 sequence_num, s64 compl_ret, void *context); + /** * Pre-pinned host memory support * diff --git a/neuron_ds.c b/neuron_ds.c index fc63e18..85b989c 100644 --- a/neuron_ds.c +++ b/neuron_ds.c @@ -145,11 +145,45 @@ static void neuron_ds_free_entry(struct neuron_datastore_entry *entry) static u64 ds_entry_clear_counter = 1; -static void neuron_ds_clear_entry(struct neuron_datastore_entry *entry) +/** + * neuron_ds_clear_entry_data_per_process - Clear per-process data from an NDS entry while + * preserving NC counters for real-time sysfs metrics. + * + * Cleared (per-process, stale after exit): + * - Header: signature/version (runtime re-initializes on next nds_open) + * - ND counters: runtime version, framework version, agg_neff_id (per-process metadata) + * - Objects: process info, model info, process ext info (per-process state) + * + * Preserved (cumulative, needed for real-time sysfs reads): + * - Primary NC counters: inference_count, nc_time_in_use, flop_count, status counters + * - Extended NC counters: hw_err_collectives, hbm_ue, etc. + * + * ND counters are cleared because they hold per-process metadata (runtime version, + * framework type) that becomes stale after exit. The next process overwrites them + * on nds_open. NC counters accumulate across processes and are summed by sysfs on read. + */ +static void neuron_ds_clear_entry_data_per_process(struct neuron_datastore_entry *entry) { + void *va = entry->mc->va; + int nc; + entry->pid = 0; entry->clear_tick = __sync_fetch_and_add(&ds_entry_clear_counter, 1); - memset(entry->mc->va, 0, NEURON_DATASTORE_SIZE); + + // Zero header + memset(va + NDS_HEADER_START, 0, NDS_HEADER_SIZE); + // Zero ND counters + memset(va + NDS_ND_COUNTERS_START, 0, NDS_ND_COUNTERS_SIZE); + // Zero objects section + memset(va + NDS_OBJECTS_START, 0, NDS_OBJECTS_SIZE); + + // Zero device mem-usage NC counters (not cumulative) + for (nc = 0; nc < NDS_MAX_NEURONCORE_COUNT; nc++) { + memset(&NDS_NEURONCORE_COUNTERS(va, nc)[NDS_NC_COUNTER_MEM_USAGE_CODE_DEVICE], 0, NDS_NC_MEM_USAGE_DEVICE_SIZE); + } + for (nc = 0; nc < NDS_EXT_MAX_NEURONCORE_COUNT; nc++) { + memset(&NDS_EXT_NEURONCORE_NC_DATA(va, nc)[NDS_NC_COUNTER_MEM_USAGE_CODE_DEVICE], 0, NDS_NC_MEM_USAGE_DEVICE_SIZE); + } } static void neuron_ds_release_entry(struct neuron_datastore *nds, @@ -159,8 +193,7 @@ static void neuron_ds_release_entry(struct neuron_datastore *nds, if (!current_pid_is_owner) return; nmetric_partial_aggregate(nds->parent, entry); - nsysfsmetric_nds_aggregate(nds->parent, entry); - neuron_ds_clear_entry(entry); + neuron_ds_clear_entry_data_per_process(entry); } void neuron_ds_release_pid(struct neuron_datastore *nds, pid_t pid) @@ -193,7 +226,7 @@ void neuron_ds_destroy(struct neuron_datastore *nds) void neuron_ds_clear(struct neuron_datastore *nds) { - neuron_ds_for_each_entry(nds, neuron_ds_clear_entry); + neuron_ds_for_each_entry(nds, neuron_ds_clear_entry_data_per_process); } // This gets the value of an NC counter in the following 3 cases: diff --git a/neuron_fw_io.c b/neuron_fw_io.c index 4bf58af..13edbf3 100644 --- a/neuron_fw_io.c +++ b/neuron_fw_io.c @@ -54,7 +54,7 @@ int fw_io_ecc_read(void *bar0, uint64_t ecc_offset, uint32_t *ecc_err_count) return 0; } -int fw_io_misc_ram_reg_read(void *bar0, u64 offset, u32 *val) +int fw_io_misc_ram_reg_read(void *bar0, u64 offset, u32 *val, bool log_error) { if (offset % 4 != 0) { pr_err("invalid misc ram offset, needs to be 4 byte aligned\n"); @@ -62,8 +62,10 @@ int fw_io_misc_ram_reg_read(void *bar0, u64 offset, u32 *val) } void *addr = bar0 + ndhal->ndhal_address_map.bar0_misc_ram_offset + offset; int ret = ndhal->ndhal_fw_io.fw_io_read_csr_array(&addr, val, 1, true); - if (ret) { - pr_err("failed to read misc ram reg at offset 0x%llx\n", offset); + if (ret || *val == 0xdeadbeef) { + if (log_error) { + pr_err("failed to read misc ram reg at offset 0x%llx\n", offset); + } return -EIO; } return 0; @@ -866,9 +868,9 @@ int fw_io_get_performance_profile(struct fw_io_ctx *ctx, uint32_t *profile) return -EINVAL; } - req.type = 1; + req.type = FW_IO_GET_DATA_PERF_PROFILE; - ret = fw_io_execute_request_new(ctx, FW_IO_CMD_GET_DATA, (u8 *)&req, sizeof(req), (u8 *)&resp, sizeof(resp)); + ret = fw_io_execute_request_new(ctx, FW_IO_CMD_GET_DATA, (u8 *)&req, sizeof(req.type), (u8 *)&resp, sizeof(resp)); if (ret == 0) { *profile = (uint32_t)resp.profile; } else { @@ -904,17 +906,17 @@ int fw_io_enable_throttling_notifications(struct fw_io_ctx *ctx, bool enable) int fw_io_get_available_profiles(struct fw_io_ctx *ctx, u16 feature, u8 *num_profiles, u8 bitmap[32]) { - struct fw_io_get_available_profiles_request req; + struct fw_io_get_data_request req = {0}; struct fw_io_get_available_profiles_response response; int ret; if (!ctx) { return -EINVAL; } - req.type = 2; - req.operation = feature; + req.type = FW_IO_GET_DATA_AVAILABLE_PROFILES; + req.available_profiles.operation = feature; - ret = fw_io_execute_request_new(ctx, FW_IO_CMD_GET_DATA, (u8*)&req, sizeof(req), (u8*)&response, sizeof(response)); + ret = fw_io_execute_request_new(ctx, FW_IO_CMD_GET_DATA, (u8*)&req, sizeof(req.type) + sizeof(req.available_profiles), (u8*)&response, sizeof(response)); if (ret) { return ret; } @@ -923,3 +925,34 @@ int fw_io_get_available_profiles(struct fw_io_ctx *ctx, u16 feature, u8 *num_pro memcpy(bitmap, response.profiles_bitmap, sizeof(response.profiles_bitmap)); return 0; } + +int fw_io_get_bar_info(struct fw_io_ctx *ctx, u8 query_type, struct fw_io_bar_entry *entries, u8 *count) +{ + struct fw_io_get_data_request req = {0}; + struct fw_io_bar_entry_response resp[NEURON_MAX_SWITCH_BARS] = {0}; + int ret = 0; + int i = 0; + + if (!ctx) { + return -EINVAL; + } + + req.type = FW_IO_GET_DATA_BAR_INFO; + req.bar_info.query_type = query_type; + + ret = fw_io_execute_request_new(ctx, FW_IO_CMD_GET_DATA, (u8 *)&req, sizeof(req.type) + sizeof(req.bar_info), (u8 *)resp, sizeof(resp)); + if (ret) { + return ret; + } + + for (i = 0; i < NEURON_MAX_SWITCH_BARS; i++) { + if (resp[i].bar_type == 0) { + break; + } + entries[i].bar_type = resp[i].bar_type; + entries[i].bar_address = (u64)resp[i].bar_address << 32; + } + *count = i; + + return 0; +} diff --git a/neuron_fw_io.h b/neuron_fw_io.h index ac91b98..6c5924e 100644 --- a/neuron_fw_io.h +++ b/neuron_fw_io.h @@ -76,9 +76,23 @@ union fw_io_req_perfprofile_data { uint32_t raw[2]; }; +enum fw_io_get_data_type { + FW_IO_GET_DATA_PERF_PROFILE = 0x01, + FW_IO_GET_DATA_AVAILABLE_PROFILES = 0x02, + FW_IO_GET_DATA_BAR_INFO = 0x10, +}; + struct fw_io_get_data_request { uint8_t type; -}; + union { + struct { + uint16_t operation; + } available_profiles; + struct { + uint8_t query_type; + } bar_info; + }; +} __packed; struct fw_io_get_perfprofile_response { uint8_t reserved[4]; @@ -88,16 +102,25 @@ struct fw_io_get_perfprofile_response { uint8_t ocw; }; -struct fw_io_get_available_profiles_request { - uint8_t type; // must be 2 - uint16_t operation; -} __packed; - struct fw_io_get_available_profiles_response { uint8_t num_profiles; - uint8_t profiles_bitmap[32]; + uint8_t profiles_bitmap[32]; +}; + +#define NEURON_MAX_SWITCH_BARS 16 + +struct fw_io_bar_entry_response { + uint8_t bar_type; + uint8_t reserved[3]; + uint32_t bar_address; +} __packed; + +struct fw_io_bar_entry { + uint8_t bar_type; + uint64_t bar_address; }; + enum fw_io_get_available_profiles_feature { FW_IO_AVAILABLE_PERF_PROFILES_ALL = 0, FW_IO_AVAILABLE_PERF_PROFILES_HBM_7200 = 5 @@ -145,6 +168,8 @@ enum { // - The value of this register is used to determine the offset of other registers. FW_IO_REG_API_VERSION_OFFSET = 0x00, + FW_IO_REG_HEALTH_CHECK_STATUS_OFFSET = 0x04, // reg 1 + FW_IO_REG_HEALTH_CHECK_SEQ_OFFSET = 0x0C, // reg 3 // MISC RAM instance/partition size info // (0:5) instance size, 16:30 partition size, 31 partition size valid @@ -243,6 +268,13 @@ enum { #define _REG_SERVERINFO_RVALIDMASK ((1 << _REG_SERVERINFO_RVALIDBITS)-1) #define _REG_SERVERINFO_RVALID(rval) (((rval) >> _REG_SERVERINFO_RVALIDSHIFT) & _REG_SERVERINFO_RVALIDMASK) +// healthcheck regs - Reg 3 field extraction +#define _REG_HEALTHCHECK_HEARTBEATBITS 8 +#define _REG_HEALTHCHECK_HEARTBEATSHIFT 12 +#define _REG_HEALTHCHECK_HEARTBEATMASK ((1 << _REG_HEALTHCHECK_HEARTBEATBITS)-1) +#define _REG_HEALTHCHECK_HEARTBEAT(val) (((val) >> _REG_HEALTHCHECK_HEARTBEATSHIFT) & _REG_HEALTHCHECK_HEARTBEATMASK) + + // #define FW_IO_REG_METRIC_BUF_SZ 128 @@ -507,9 +539,10 @@ int fw_io_ecc_read(void *bar0, uint64_t ecc_offset, uint32_t *ecc_err_count); * @param bar0: mapped BAR0 base * @param offset: byte offset of the register within the misc RAM block (e.g., FW_IO_REG_*_OFFSET) * @param val: output register value + * @param log_error: whether to log an error in dmesg (false for periodic read which will create too much noise) * @return 0 on success */ -int fw_io_misc_ram_reg_read(void *bar0, u64 offset, u32 *val); +int fw_io_misc_ram_reg_read(void *bar0, u64 offset, u32 *val, bool log_error); /** * fw_io_serial_number_read() - Read serial number @@ -591,4 +624,13 @@ int fw_io_enable_throttling_notifications(struct fw_io_ctx *ctx, bool enable); */ int fw_io_get_available_profiles(struct fw_io_ctx *ctx, u16 feature, u8 *num_profiles, u8 bitmap[32]); +/** + * fw_io_get_bar_info() - Get switch fabric BAR info + * @param ctx: FWIO context + * @param query_type: 1 (INTER_SERVER) or 2 (INTRA_SERVER) + * @param entries: BAR entries in response + * @param count: Number of valid entries in response + */ +int fw_io_get_bar_info(struct fw_io_ctx *ctx, u8 query_type, struct fw_io_bar_entry *entries, u8 *count); + #endif diff --git a/neuron_ioctl.h b/neuron_ioctl.h index 095dbb4..7b42bc7 100644 --- a/neuron_ioctl.h +++ b/neuron_ioctl.h @@ -143,6 +143,17 @@ struct neuron_ioctl_mem_copy64 { __u64 size; // [in] Size of the transfer. __u64 src_offset; // [in] Offset in the source memory handle. __u64 dst_offset; // [in] Offset in the destination memory handle. + __u64 sequence_num; // [in] The sequence number that uniquely identifies each async I/O. + void *context; // [in] Opaque context pointer passed back in the completion queue. +}; + +// Old drivers use the same mem copy ioctl number with this smaller payload. +struct neuron_ioctl_mem_copy64_deprecated { + __u64 src_mem_handle; // [in] Source memory handle from where data is copied. + __u64 dst_mem_handle; // [in] Destination memory handle to data is to be copied. + __u64 size; // [in] Size of the transfer. + __u64 src_offset; // [in] Offset in the source memory handle. + __u64 dst_offset; // [in] Offset in the destination memory handle. }; struct neuron_ioctl_mem_copy_async { @@ -706,6 +717,14 @@ struct neuron_ioctl_host_mem_unpin { __u64 va; // [in] VA to unpin (must match exact VA from pin) }; +#define NEURON_IOCTL_MAX_BAR_ENTRIES 16 +struct neuron_ioctl_get_bar_info { + __u32 query_type; // [in] 1=INTER_SERVER, 2=INTRA_SERVER + __u32 count; // [out] num valid entries + __u32 bar_types[NEURON_IOCTL_MAX_BAR_ENTRIES]; // [out] enum neuron_switch_fabric_bar_type + __u64 bars[NEURON_IOCTL_MAX_BAR_ENTRIES]; // [out] 64-bit PCIe BAR address +}; + #define NEURON_IOCTL_BASE 'N' /* Deprecated reset related IOCTLs. Now it would always return success. */ @@ -737,6 +756,7 @@ struct neuron_ioctl_host_mem_unpin { #define NEURON_IOCTL_MEM_FREE _IOR(NEURON_IOCTL_BASE, 22, struct neuron_ioctl_mem_free *) /** Copy data between two memory handles. (using DMA) */ #define NEURON_IOCTL_MEM_COPY _IOR(NEURON_IOCTL_BASE, 23, struct neuron_ioctl_mem_copy *) +#define NEURON_IOCTL_MEM_COPY64_DEPRECATED _IOR(NEURON_IOCTL_BASE, 23, struct neuron_ioctl_mem_copy64_deprecated) #define NEURON_IOCTL_MEM_COPY64 _IOR(NEURON_IOCTL_BASE, 23, struct neuron_ioctl_mem_copy64) /** Copy data from/to given host buffer to/from memory_handle. (using DMA)*/ @@ -935,4 +955,7 @@ struct neuron_ioctl_host_mem_unpin { /** Get dma-buf file-descriptor with offset (v2) */ #define NEURON_IOCTL_DMABUF_FD_V2 _IOR(NEURON_IOCTL_BASE, 138, struct neuron_ioctl_dmabuf_fd_v2 *) +/** Get switch fabric BAR info for a device */ +#define NEURON_IOCTL_GET_BAR_INFO _IOWR(NEURON_IOCTL_BASE, 139, struct neuron_ioctl_get_bar_info) + #endif diff --git a/neuron_log.c b/neuron_log.c index 3e5704f..b0a4f2d 100644 --- a/neuron_log.c +++ b/neuron_log.c @@ -16,6 +16,9 @@ #include "neuron_device.h" #define NEURON_LOG_NUM_ENTRIES 1024 +#define NEURON_LOG_INDEX_MASK (NEURON_LOG_NUM_ENTRIES - 1) +static_assert(NEURON_LOG_NUM_ENTRIES && !(NEURON_LOG_NUM_ENTRIES & (NEURON_LOG_NUM_ENTRIES - 1)), + "NEURON_LOG_NUM_ENTRIES must be a power of 2 for mask indexing"); static const char * neuron_log_rec_type_to_str( enum neuron_log_type type) { @@ -105,13 +108,17 @@ int neuron_log_dump(struct neuron_device *nd, pid_t pid, uint32_t log_dump_limit } // grab a copy of the log - tail_index = (atomic_read(&nd->log_obj.tail) - 1) % NEURON_LOG_NUM_ENTRIES; + // Note: atomic_t is a signed int. After 2^31 ioctls log_obj.tail wraps + // negative and signed '%' on a negative dividend yields a negative result, + // which when assigned to uint32_t becomes ~4.29B and causes the print loop + // below to never terminate. Cast to unsigned and use a power-of-2 bitmask. + tail_index = ((unsigned int)atomic_read(&nd->log_obj.tail) - 1) & NEURON_LOG_INDEX_MASK; memcpy(log_snapshot, nd->log_obj.log, sizeof(struct neuron_log_rec) * NEURON_LOG_NUM_ENTRIES); // scan backwards // for (i=1, j=0; i < NEURON_LOG_NUM_ENTRIES-1; i++) { - struct neuron_log_rec * log_rec = &log_snapshot[(tail_index-i) % NEURON_LOG_NUM_ENTRIES]; + struct neuron_log_rec * log_rec = &log_snapshot[(tail_index-i) & NEURON_LOG_INDEX_MASK]; if ((log_rec->type == NEURON_LOG_TYPE_INVALID) || (pid && log_rec->pid != pid)) { continue; @@ -128,11 +135,11 @@ int neuron_log_dump(struct neuron_device *nd, pid_t pid, uint32_t log_dump_limit // print forwards // - i = (tail_index-i) % NEURON_LOG_NUM_ENTRIES; + i = (tail_index-i) & NEURON_LOG_INDEX_MASK; while (i != tail_index) { struct neuron_log_rec * log_rec = &log_snapshot[i]; - i = (i+1) % NEURON_LOG_NUM_ENTRIES; + i = (i+1) & NEURON_LOG_INDEX_MASK; if ((log_rec->type == NEURON_LOG_TYPE_INVALID) || (pid && log_rec->pid != pid)) { continue; } diff --git a/neuron_metrics.c b/neuron_metrics.c index a3a8914..9b65153 100644 --- a/neuron_metrics.c +++ b/neuron_metrics.c @@ -1013,6 +1013,7 @@ static void nmetric_start_new_session(struct neuron_device *nd, u64 *curr_metric static void nmetric_sample_high_freq(struct neuron_device *nd) { npower_sample_utilization(nd); + nsysfsmetric_health_status_tick(nd); } static void nmetric_aggregate_and_post_tick(struct neuron_device *nd, struct nmetric_versions *component_versions, u64 *curr_feature_bitmap, u64 *freed_feature_bitmap, u64 *const_u64_metrics, u64 *freed_const_u64_metrics, u8 tick) @@ -1059,8 +1060,6 @@ static int nmetric_thread_fn(void *arg) u64 last_metric_post_time; u64 start_jiffies = jiffies; u64 current_slow_tick; - u64 last_health_tick_jiffies = jiffies; - const u64 health_tick_interval_jiffies = msecs_to_jiffies(60 * 1000); // health_status cache refresh cadence u8 tick_budget = 0; // how many ticks can be posted in a certain iteration of the loop // initialize all aggregation buffers @@ -1098,12 +1097,6 @@ static int nmetric_thread_fn(void *arg) // There are some metrics that we sample at a relatively higher frequency. Do that here. nmetric_sample_high_freq(nd); - // Refresh health_status cached sysfs values - if ((jiffies - last_health_tick_jiffies) >= health_tick_interval_jiffies) { - nsysfsmetric_health_status_tick(nd); - last_health_tick_jiffies = jiffies; - } - // For the slower metrics, we want to log once every post_delay_in_jiffies jiffies. // We track this by keeping track of the number of intervals since this thread started // up so that we don't introduce drift due to the latency of other loop operations. diff --git a/neuron_mmap.h b/neuron_mmap.h index a0cb3e3..50c0dce 100644 --- a/neuron_mmap.h +++ b/neuron_mmap.h @@ -19,10 +19,20 @@ #endif /* - * Linux 6.10 removed get_unmapped_area from mm_struct and replaced it - * with the standalone mm_get_unmapped_area() function. + * mm_get_unmapped_area() signature history: + * < 6.10: no standalone function; call current->mm->get_unmapped_area() + * directly (5 args). + * 6.10: introduced as + * mm_get_unmapped_area(struct mm_struct *mm, struct file *filp, + * addr, len, pgoff, flags) -- 6 args. + * 6.19: the leading mm parameter was dropped, reverting to + * mm_get_unmapped_area(struct file *filp, addr, len, pgoff, flags) + * -- 5 args. (Present in v6.18, gone in v6.19.) */ -#if (!defined(RHEL_RELEASE_CODE) && (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 10, 0))) || (defined(RHEL_RELEASE_CODE) && (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(9, 5))) +#if (!defined(RHEL_RELEASE_CODE) && (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 19, 0))) +#define nmmap_kern_get_unmapped_area(filep, addr, len, pgoff, flags) \ + mm_get_unmapped_area(filep, addr, len, pgoff, flags) +#elif (!defined(RHEL_RELEASE_CODE) && (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 10, 0))) || (defined(RHEL_RELEASE_CODE) && (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(9, 5))) #define nmmap_kern_get_unmapped_area(filep, addr, len, pgoff, flags) \ mm_get_unmapped_area(current->mm, filep, addr, len, pgoff, flags) #else diff --git a/neuron_module.c b/neuron_module.c index 48feed0..aa56d91 100644 --- a/neuron_module.c +++ b/neuron_module.c @@ -20,13 +20,13 @@ #include "neuron_dma.h" #include "neuron_test.h" -MODULE_DESCRIPTION("Neuron Driver, built from SHA: c64fcb38034831fa1d30e95aca96ff5ed5005726"); +MODULE_DESCRIPTION("Neuron Driver, built from SHA: 40a502a5d7e0f165b7e4d89588eb1da01c068e8e"); MODULE_LICENSE("GPL"); -MODULE_VERSION("2.29.0.0"); +MODULE_VERSION("2.30.2.0"); MODULE_ALIAS("pci:v00001d0fd00007064sv*sd*bc*sc*i*"); -const char driver_version[] = "2.29.0.0"; -const char driver_revision[] = "c64fcb38034831fa1d30e95aca96ff5ed5005726"; +const char driver_version[] = "2.30.2.0"; +const char driver_revision[] = "40a502a5d7e0f165b7e4d89588eb1da01c068e8e"; #ifdef CONFIG_FAULT_INJECTION diff --git a/neuron_nq.c b/neuron_nq.c index 932522a..5dc1f26 100644 --- a/neuron_nq.c +++ b/neuron_nq.c @@ -43,7 +43,7 @@ static int nnq_halt(struct neuron_device *nd, u8 nc_id, u8 eng_index, u32 nq_typ } ndhal->ndhal_nq.nnq_set_hwaddr(nd, nc_id, eng_index, nq_type, 0, 0); - + return 0; } @@ -157,8 +157,9 @@ void nnq_destroy_nc(struct neuron_device *nd, u8 nc_id) u8 eng_index; u8 nq_type; u8 ts_id; + u8 nq_queue_count = ndhal->ndhal_nq.nnq_get_nq_queue_count(); - for (eng_index = 0; eng_index < MAX_NQ_ENGINE; eng_index++) { + for (eng_index = 0; eng_index < nq_queue_count; eng_index++) { for (nq_type = 0; nq_type < MAX_NQ_TYPE; nq_type++) { nnq_halt(nd, nc_id, eng_index, nq_type); } @@ -167,7 +168,7 @@ void nnq_destroy_nc(struct neuron_device *nd, u8 nc_id) // wait for halted notific queues to drain msleep(1); - for (eng_index = 0; eng_index < MAX_NQ_ENGINE; eng_index++) { + for (eng_index = 0; eng_index < nq_queue_count; eng_index++) { for (nq_type = 0; nq_type < MAX_NQ_TYPE; nq_type++) { nnq_destroy(nd, nc_id, eng_index, nq_type); } diff --git a/neuron_ring.c b/neuron_ring.c index a092acf..4579416 100644 --- a/neuron_ring.c +++ b/neuron_ring.c @@ -818,6 +818,14 @@ static void ndmar_h2t_ring_free_all(struct neuron_device *nd, int nc_idx) for (qid = 0; qid < ndhal->ndhal_udma.num_queues; qid++) { queue = ndmar_get_queue(eng, qid); ring = ndmar_get_ring(queue); + + // The default queues do not go through the ndmar_h2t_ring_request path + // So they should be handled specially at cleanup too + if (qid == ndhal->ndhal_ndmar.ndmar_get_h2t_def_qid(nc_idx)) { + ndmar_h2t_ring_free(eng, ring); + continue; + } + if (ndmar_h2t_ring_is_allocated(ring) && ring->h2t_nc_id == nc_idx) { if (ndmar_h2t_ring_is_h2t(ring)) { // h2t queue free all resources diff --git a/neuron_sysfs_metrics.c b/neuron_sysfs_metrics.c index 33381e9..9212ef3 100644 --- a/neuron_sysfs_metrics.c +++ b/neuron_sysfs_metrics.c @@ -145,7 +145,9 @@ static const nsysfsmetric_attr_info_t arch_info_attrs_info_tbl[] = { static const int arch_info_attrs_info_tbl_cnt = sizeof(arch_info_attrs_info_tbl) / sizeof(nsysfsmetric_attr_info_t); static const nsysfsmetric_attr_info_t ecc_attrs_info_tbl[] = { + ATTR_INFO("sram_ecc_corrected", NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_COUNTER_ECC_SRAM_CORRECTED), OTHER), ATTR_INFO("sram_ecc_uncorrected", NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_COUNTER_ECC_SRAM_UNCORRECTED), OTHER), + ATTR_INFO("mem_ecc_corrected", NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_COUNTER_ECC_HBM_CORRECTED), OTHER), ATTR_INFO("mem_ecc_uncorrected", NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_COUNTER_ECC_HBM_UNCORRECTED), OTHER), ATTR_INFO("mem_ecc_repairable_uncorrected", NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_COUNTER_ECC_REPAIRABLE_HBM_UNCORRECTED), OTHER), }; @@ -158,20 +160,34 @@ struct health_status_reg_map { }; static const struct health_status_reg_map health_status_reg_tbl[] = { - { HEALTH_STATUS_SLOT_SRAM_ECC, FW_IO_REG_SRAM_ECC_OFFSET, true }, - { HEALTH_STATUS_SLOT_HBM0_ECC, FW_IO_REG_HBM0_ECC_OFFSET, true }, - { HEALTH_STATUS_SLOT_HBM1_ECC, FW_IO_REG_HBM1_ECC_OFFSET, true }, - { HEALTH_STATUS_SLOT_HBM2_ECC, FW_IO_REG_HBM2_ECC_OFFSET, true }, - { HEALTH_STATUS_SLOT_HBM3_ECC, FW_IO_REG_HBM3_ECC_OFFSET, true }, - { HEALTH_STATUS_SLOT_HBM_REPAIR_STATE, FW_IO_REG_HBM_REPAIR_STATE_OFFSET, true }, - { HEALTH_STATUS_SLOT_FW_API_VERSION, FW_IO_REG_API_VERSION_OFFSET, false }, + { HEALTH_STATUS_SLOT_HEALTH_STATUS_CHECK, FW_IO_REG_HEALTH_CHECK_STATUS_OFFSET, true }, + { HEALTH_STATUS_SLOT_HEALTH_STATUS_SEQ, FW_IO_REG_HEALTH_CHECK_SEQ_OFFSET, false }, }; static const int health_status_reg_tbl_cnt = sizeof(health_status_reg_tbl) / sizeof(health_status_reg_tbl[0]); +struct healthcheck_status_field { + u8 shift; + u8 mask; + const char *name; +}; + +static const struct healthcheck_status_field healthcheck_status_fields[] = { + { 3, 0x1, "hbm" }, + { 4, 0x1, "firmware_service_api" }, + { 5, 0x1, "firmware_service_processor" }, + { 6, 0x1, "sram" }, + { 7, 0x1, "link_intraserver" }, + { 8, 0x1, "link_host" }, + { 9, 0x1, "link_efa" }, + { 10, 0x1, "link_interserver" }, + { 11, 0x1, "link_switch" }, + { 12, 0x1, "axi_fabric" }, +}; +static const int healthcheck_status_fields_cnt = sizeof(healthcheck_status_fields) / sizeof(healthcheck_status_fields[0]); + static const nsysfsmetric_attr_info_t health_status_attrs_info_tbl[] = { - ATTR_INFO("hbm_ecc_err_count", NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_HEALTH_STATUS_HBM_UE_COUNT), CACHED_VALUES), - ATTR_INFO("repairable_hbm_ecc_err_count", NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_HEALTH_STATUS_REPAIRABLE_HBM_UE_COUNT), CACHED_VALUES), - ATTR_INFO("sram_ecc_err_count", NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_HEALTH_STATUS_SRAM_UE_COUNT), CACHED_VALUES), + ATTR_INFO("unhealthy_components", NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_HEALTH_STATUS_OVERALL_HEALTH), CACHED_VALUES), + ATTR_INFO("healthcheck_heartbeat", NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_HEALTH_STATUS_HEARTBEAT), CACHED_VALUES), ATTR_INFO("hw_error_event", NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_HEALTH_STATUS_HW_ERROR_EVENT), CACHED_VALUES), }; static const int health_status_attrs_info_tbl_cnt = sizeof(health_status_attrs_info_tbl) / sizeof(nsysfsmetric_attr_info_t); @@ -284,13 +300,36 @@ static void nsysfsmetric_get_neuron_architecture(struct nsysfsmetric_metrics *sy snprintf(arch, NEURON_ARCH_MAX_LEN, "%s%s", arch_prefix, arch_suffix); } +static bool nsysfsmetric_is_nds_metric(int metric_id) +{ + return metric_id >= NDS_NC_COUNTER_ID_TO_SYSFS_METRIC_ID(0) && metric_id < (NDS_NC_COUNTER_ID_TO_SYSFS_METRIC_ID(0) + NDS_COUNTER_COUNT); +} + +static u64 nsysfsmetric_sum_nds_counter(struct neuron_device *nd, int nc_id, int metric_id) +{ + struct neuron_datastore *nds = &nd->datastore; + int ds_id = SYSFS_METRIC_ID_TO_NDS_NC_COUNTER_ID(metric_id); + u64 val = 0; + int i = 0; + + for (i = 0; i < NEURON_MAX_DATASTORE_ENTRIES_PER_DEVICE; i++) { + struct mem_chunk *mc = nds->entries[i].mc; + if (!mc || !mc->va) + continue; + val += get_neuroncore_counter_value(&nds->entries[i], nc_id, ds_id); + } + + // One MAC = two floating point operations (multiply + add) + if (ds_id == NDS_NC_COUNTER_MAC_COUNT) + val *= 2; + + return val; +} static ssize_t nsysfsmetric_show_nrt_total_metrics(struct nsysfsmetric_metrics *sysfs_metrics, struct metric_attribute *attr, char *buf) { - ssize_t len = 0; - if (attr->metric_id < 0 || attr->metric_id >= MAX_METRIC_ID || attr->nc_id >= MAX_NC_PER_DEVICE) { pr_err("invalid metric_id %d or nc_id %d of attr_type TOTAL\n", attr->metric_id, attr->nc_id); return 0; @@ -298,14 +337,17 @@ static ssize_t nsysfsmetric_show_nrt_total_metrics(struct nsysfsmetric_metrics * u64 val = 0; - if (attr->nc_id == -1) { - val = sysfs_metrics->nrt_nd_metrics[attr->metric_id].total; + if (nsysfsmetric_is_nds_metric(attr->metric_id) && attr->nc_id >= 0) { + struct neuron_device *nd = sysfs_metrics->nd; + val = nsysfsmetric_sum_nds_counter(nd, attr->nc_id, attr->metric_id); } else { - val = sysfs_metrics->nrt_metrics[attr->metric_id][attr->nc_id].total; + if (attr->nc_id == -1) + val = sysfs_metrics->nrt_nd_metrics[attr->metric_id].total; + else + val = sysfs_metrics->nrt_metrics[attr->metric_id][attr->nc_id].total; } - len = nsysfsmetric_sysfs_emit(buf, "%llu\n", val); - return len; + return nsysfsmetric_sysfs_emit(buf, "%llu\n", val); } static ssize_t nsysfsmetric_show_nrt_present_metrics(struct nsysfsmetric_metrics *sysfs_metrics, @@ -319,6 +361,13 @@ static ssize_t nsysfsmetric_show_nrt_present_metrics(struct nsysfsmetric_metrics return 0; } + // Return the same value as total for now. + // TODO: deprecate "present" for NDS metrics in a future release. + if (nsysfsmetric_is_nds_metric(attr->metric_id) && attr->nc_id >= 0) { + struct neuron_device *nd = sysfs_metrics->nd; + return nsysfsmetric_sysfs_emit(buf, "%llu\n", nsysfsmetric_sum_nds_counter(nd, attr->nc_id, attr->metric_id)); + } + u64 val = 0; if (attr->nc_id == -1) { val = sysfs_metrics->nrt_nd_metrics[attr->metric_id].present; @@ -370,7 +419,7 @@ static ssize_t nsysfsmetric_show_nrt_other_metrics(struct nsysfsmetric_metrics * else len = nsysfsmetric_sysfs_emit(buf, "-1\n"); } else if (attr->metric_id == NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_COUNTER_ECC_SRAM_UNCORRECTED)) { - struct neuron_device *nd = container_of(sysfs_metrics, struct neuron_device, sysfs_metrics); + struct neuron_device *nd = sysfs_metrics->nd; uint64_t ecc_offset = FW_IO_REG_SRAM_ECC_OFFSET; uint32_t ecc_err_count = 0; int ret = fw_io_ecc_read(nd->npdev.bar0, ecc_offset, &ecc_err_count); @@ -381,15 +430,39 @@ static ssize_t nsysfsmetric_show_nrt_other_metrics(struct nsysfsmetric_metrics * ecc_err_count = 0; } len = nsysfsmetric_sysfs_emit(buf, "%u\n", ecc_err_count & 0x0000ffff); + } else if (attr->metric_id == NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_COUNTER_ECC_SRAM_CORRECTED)) { + struct neuron_device *nd = sysfs_metrics->nd; + uint64_t ecc_offset = FW_IO_REG_SRAM_ECC_OFFSET; + uint32_t ecc_err_count = 0; + int ret = fw_io_ecc_read(nd->npdev.bar0, ecc_offset, &ecc_err_count); + if (ret) { + ecc_err_count = 0; + pr_err("sysfs failed to read ECC SRAM error from FWIO\n"); + } else if (ecc_err_count == 0xdeadbeef) { + ecc_err_count = 0; + } + len = nsysfsmetric_sysfs_emit(buf, "%u\n", (ecc_err_count >> 16) & 0xffff); + } else if (attr->metric_id == NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_COUNTER_ECC_HBM_CORRECTED)) { + struct neuron_device *nd = sysfs_metrics->nd; + uint64_t ecc_offset = FW_IO_REG_HBM0_ECC_OFFSET; // reg 17 is for all HBMs now, earlier was only for HBM0 + uint32_t ecc_err_count = 0; + int ret = fw_io_ecc_read(nd->npdev.bar0, ecc_offset, &ecc_err_count); + if (ret) { + ecc_err_count = 0; + pr_err("sysfs failed to read ECC HBM error from FWIO\n"); + } else if (ecc_err_count == 0xdeadbeef) { + ecc_err_count = 0; + } + len = nsysfsmetric_sysfs_emit(buf, "%u\n", (ecc_err_count >> 16) & 0xffff); } else if (attr->metric_id == NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_COUNTER_ECC_HBM_UNCORRECTED) || attr->metric_id == NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_COUNTER_ECC_REPAIRABLE_HBM_UNCORRECTED)) { - struct neuron_device *nd = container_of(sysfs_metrics, struct neuron_device, sysfs_metrics); + struct neuron_device *nd = sysfs_metrics->nd; uint32_t err_count; bool get_repairable = (attr->metric_id == NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_COUNTER_ECC_REPAIRABLE_HBM_UNCORRECTED)); ndhal->ndhal_sysfs_metrics.nsysfsmetric_get_hbm_error_count(nd, get_repairable, &err_count); len = nsysfsmetric_sysfs_emit(buf, "%u\n", err_count); } else if (attr->metric_id == NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_OTHER_SERIAL_NUMBER)) { - struct neuron_device *nd = container_of(sysfs_metrics, struct neuron_device, sysfs_metrics); + struct neuron_device *nd = sysfs_metrics->nd; uint64_t serial_number = 0; int ret = fw_io_serial_number_read(nd->npdev.bar0, &serial_number); if (ret @@ -402,7 +475,7 @@ static ssize_t nsysfsmetric_show_nrt_other_metrics(struct nsysfsmetric_metrics * len = nsysfsmetric_sysfs_emit(buf, "%016llx\n", serial_number); } } else if (attr->metric_id == NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_OTHER_POWER_UTILIZATION)) { - struct neuron_device *nd = container_of(sysfs_metrics, struct neuron_device, sysfs_metrics); + struct neuron_device *nd = sysfs_metrics->nd; char buffer[256]; int ret = npower_format_stats(nd, buffer, 256); @@ -411,7 +484,7 @@ static ssize_t nsysfsmetric_show_nrt_other_metrics(struct nsysfsmetric_metrics * } len = nsysfsmetric_sysfs_emit(buf, "%s\n", buffer); } else if (attr->metric_id == NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_OTHER_POWER_UTILIZATION_RAW)) { - struct neuron_device *nd = container_of(sysfs_metrics, struct neuron_device, sysfs_metrics); + struct neuron_device *nd = sysfs_metrics->nd; char buffer[256]; int ret = npower_format_raw(nd, buffer, sizeof(buffer)); @@ -420,7 +493,7 @@ static ssize_t nsysfsmetric_show_nrt_other_metrics(struct nsysfsmetric_metrics * } len = nsysfsmetric_sysfs_emit(buf, "%s", buffer); } else if (attr->metric_id == NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_COUNTER_PE_ARRAY_ACTIVITY)) { - struct neuron_device *nd = container_of(sysfs_metrics, struct neuron_device, sysfs_metrics); + struct neuron_device *nd = sysfs_metrics->nd; char buffer[256]; int ret = ndhal->ndhal_tpb.pe_format_activity_stats(nd, attr->nc_id, buffer, sizeof(buffer)); @@ -442,31 +515,37 @@ static ssize_t nsysfsmetric_show_cached_values_metrics(struct nsysfsmetric_metri u32 value = 0; switch (attr->metric_id) { - case NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_HEALTH_STATUS_SRAM_UE_COUNT): - value = READ_ONCE(sysfs_metrics->cached_health_regs[HEALTH_STATUS_SLOT_SRAM_ECC]); - return nsysfsmetric_sysfs_emit(buf, "%u\n", value & 0xffff); // Lower 16 bits - case NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_HEALTH_STATUS_HBM_UE_COUNT): - // TODO: Use cached HEALTH_STATUS_SLOT_FW_API_VERSION - // For now, safe to assume api_version >= 6 - value = 0; - value += (((READ_ONCE(sysfs_metrics->cached_health_regs[HEALTH_STATUS_SLOT_HBM0_ECC])) >> 12) & 0xf); - value += (((READ_ONCE(sysfs_metrics->cached_health_regs[HEALTH_STATUS_SLOT_HBM1_ECC])) >> 12) & 0xf); - value += (((READ_ONCE(sysfs_metrics->cached_health_regs[HEALTH_STATUS_SLOT_HBM2_ECC])) >> 12) & 0xf); - value += (((READ_ONCE(sysfs_metrics->cached_health_regs[HEALTH_STATUS_SLOT_HBM3_ECC])) >> 12) & 0xf); - if ((READ_ONCE(sysfs_metrics->cached_health_regs[HEALTH_STATUS_SLOT_HBM_REPAIR_STATE]) & 0x3) == 0x2) - value +=1; - - return nsysfsmetric_sysfs_emit(buf, "%u\n", value); - case NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_HEALTH_STATUS_REPAIRABLE_HBM_UE_COUNT): - value = 0; - value += ((READ_ONCE(sysfs_metrics->cached_health_regs[HEALTH_STATUS_SLOT_HBM0_ECC])) & 0xfff); - value += ((READ_ONCE(sysfs_metrics->cached_health_regs[HEALTH_STATUS_SLOT_HBM1_ECC])) & 0xfff); - value += ((READ_ONCE(sysfs_metrics->cached_health_regs[HEALTH_STATUS_SLOT_HBM2_ECC])) & 0xfff); - value += ((READ_ONCE(sysfs_metrics->cached_health_regs[HEALTH_STATUS_SLOT_HBM3_ECC])) & 0xfff); - if ((READ_ONCE(sysfs_metrics->cached_health_regs[HEALTH_STATUS_SLOT_HBM_REPAIR_STATE]) & 0x3) == 0x1) - value +=1; - - return nsysfsmetric_sysfs_emit(buf, "%u\n", value); + case NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_HEALTH_STATUS_OVERALL_HEALTH): { + char tmp[512]; + int i, pos = 0; + u32 reg1; + + if (READ_ONCE(sysfs_metrics->health_check_regs_read_failed)) + return nsysfsmetric_sysfs_emit(buf, "device_unreachable\n"); + + reg1 = READ_ONCE(sysfs_metrics->cached_health_regs[HEALTH_STATUS_SLOT_HEALTH_STATUS_CHECK]); + for (i = 0; i < healthcheck_status_fields_cnt; i++) { + if ((reg1 >> healthcheck_status_fields[i].shift) & healthcheck_status_fields[i].mask) { + if (pos) { + pos += scnprintf(tmp + pos, sizeof(tmp) - pos, ","); + } + pos += scnprintf(tmp + pos, sizeof(tmp) - pos, "%s", healthcheck_status_fields[i].name); + } + } + + if (pos == 0) + return nsysfsmetric_sysfs_emit(buf, "\n"); + + return nsysfsmetric_sysfs_emit(buf, "%s\n", tmp); + } + + case NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_HEALTH_STATUS_HEARTBEAT): + if (READ_ONCE(sysfs_metrics->health_check_regs_read_failed)) + return nsysfsmetric_sysfs_emit(buf, "device_unreachable\n"); + + value = READ_ONCE(sysfs_metrics->cached_health_regs[HEALTH_STATUS_SLOT_HEALTH_STATUS_SEQ]); + return nsysfsmetric_sysfs_emit(buf, "%u\n", _REG_HEALTHCHECK_HEARTBEAT(value)); + case NON_NDS_ID_TO_SYSFS_METRIC_ID(NON_NDS_HEALTH_STATUS_HW_ERROR_EVENT): return nsysfsmetric_sysfs_emit(buf, "%u\n", READ_ONCE(sysfs_metrics->hw_error_event_count)); default: @@ -475,19 +554,50 @@ static ssize_t nsysfsmetric_show_cached_values_metrics(struct nsysfsmetric_metri } } +static void clear_neuroncore_counter_value(struct neuron_datastore_entry *entry, int nc_index, + int counter_index) +{ + void *ds_base_ptr = entry->mc->va; + + if (nc_index < NDS_MAX_NEURONCORE_COUNT) { + if (counter_index < NDS_NC_COUNTER_COUNT) + WRITE_ONCE(NDS_NEURONCORE_COUNTERS(ds_base_ptr, nc_index)[counter_index], 0); + else + WRITE_ONCE(NDS_EXT_NEURONCORE_COUNTERS(ds_base_ptr, nc_index)[counter_index - NDS_NC_COUNTER_COUNT], 0); + } else { + nc_index -= NDS_MAX_NEURONCORE_COUNT; + if (nc_index < NDS_EXT_MAX_NEURONCORE_COUNT && counter_index < NDS_TOTAL_NC_COUNTER_COUNT) + WRITE_ONCE(NDS_EXT_NEURONCORE_NC_DATA(ds_base_ptr, nc_index)[counter_index], 0); + } +} + static ssize_t nsysfsmetric_set_nrt_total_metrics(struct nsysfsmetric_metrics *sysfs_metrics, struct metric_attribute *attr, const char *buf, size_t size) { + struct neuron_device *nd = sysfs_metrics->nd; + int i = 0; + if (attr->metric_id < 0 || attr->metric_id >= MAX_METRIC_ID || attr->nc_id >= MAX_NC_PER_DEVICE) { pr_err("invalid metric_id %d or nc_id %d of attr_type TOTAL\n", attr->metric_id, attr->nc_id); return 0; } - if (attr->nc_id == -1) { - sysfs_metrics->nrt_nd_metrics[attr->metric_id].total = 0; + + if (nsysfsmetric_is_nds_metric(attr->metric_id) && attr->nc_id >= 0) { + int ds_id = SYSFS_METRIC_ID_TO_NDS_NC_COUNTER_ID(attr->metric_id); + for (i = 0; i < NEURON_MAX_DATASTORE_ENTRIES_PER_DEVICE; i++) { + struct mem_chunk *mc = nd->datastore.entries[i].mc; + if (!mc || !mc->va) + continue; + clear_neuroncore_counter_value(&nd->datastore.entries[i], attr->nc_id, ds_id); + } } else { - sysfs_metrics->nrt_metrics[attr->metric_id][attr->nc_id].total = 0; + if (attr->nc_id == -1) + sysfs_metrics->nrt_nd_metrics[attr->metric_id].total = 0; + else + sysfs_metrics->nrt_metrics[attr->metric_id][attr->nc_id].total = 0; } + return size; } @@ -965,7 +1075,7 @@ int nsysfsmetric_register(struct neuron_device *nd, struct kobject *neuron_devic { int ret; struct nsysfsmetric_metrics *metrics = &nd->sysfs_metrics; - + metrics->nd = nd; ret = nsysfsmetric_init_and_add_root_node(metrics, neuron_device_kobj); if (ret) { pr_err("cannot init the root node for neuron_device %d\n", nd->device_index); @@ -1098,15 +1208,21 @@ void nsysfsmetric_health_status_tick(struct neuron_device *nd) struct nsysfsmetric_metrics *metrics = &nd->sysfs_metrics; int i; bool changed = false; + bool reg_read_failed = false; if (!ndhal->ndhal_sysfs_metrics.health_status_enabled) return; for (i = 0; i < health_status_reg_tbl_cnt; i++) { u32 val; - int ret = fw_io_misc_ram_reg_read(nd->npdev.bar0, health_status_reg_tbl[i].offset, &val); - if (ret) - continue; // TODO: figure out how to communicate to sysfs readers that read failed + int ret = fw_io_misc_ram_reg_read(nd->npdev.bar0, health_status_reg_tbl[i].offset, &val, false); + if (ret) { + reg_read_failed = true; + if (!READ_ONCE(metrics->health_check_regs_read_failed)) { + changed = true; + } + continue; + } if (val != READ_ONCE(metrics->cached_health_regs[health_status_reg_tbl[i].slot])) { WRITE_ONCE(metrics->cached_health_regs[health_status_reg_tbl[i].slot], val); @@ -1116,6 +1232,12 @@ void nsysfsmetric_health_status_tick(struct neuron_device *nd) } } + if (READ_ONCE(metrics->health_check_regs_read_failed) && !reg_read_failed) { + changed = true; + } + + WRITE_ONCE(metrics->health_check_regs_read_failed, reg_read_failed); + if (changed && metrics->health_status_node) { // This function is the only writer, don't need atomic update, just volatile (READ_ONCE/WRITE_ONCE) WRITE_ONCE(metrics->hw_error_event_count, READ_ONCE(metrics->hw_error_event_count) + 1); @@ -1261,38 +1383,3 @@ void nsysfsmetric_inc_reset_req_count(struct neuron_device *nd, int nc_id) { nsysfsmetric_inc_counter(nd, NON_NDS_METRIC, NON_NDS_COUNTER_RESET_REQ_COUNT, nc_id, 1, true); } - -void nsysfsmetric_nds_aggregate(struct neuron_device *nd, struct neuron_datastore_entry *entry) -{ - int i; - int nc_id; - int ds_id; - int metric_id; - u64 delta; - void *ds_base_ptr = entry->mc->va; - - // read dynamic sysfs metric bitmap from nds - ds_id = NDS_ND_COUNTER_DYNAMIC_SYSFS_METRIC_BITMAP; - if (NDS_ND_COUNTERS(ds_base_ptr)[ds_id] > 0) { - nsysfsmetric_init_and_add_dynamic_counter_nodes(nd, NDS_ND_COUNTERS(ds_base_ptr)[ds_id]); - } - - for (nc_id = 0; nc_id < ndhal->ndhal_address_map.nc_per_device; nc_id++) { - if (((1 << nc_id) & ndhal->ndhal_address_map.dev_nc_map) == 0) { - continue; - } - // read status counters from nds - for (i = 0; i < status_counter_nodes_info_tbl_cnt; i++) { - metric_id = status_counter_nodes_info_tbl[i].metric_id; - ds_id = SYSFS_METRIC_ID_TO_NDS_NC_COUNTER_ID(metric_id); - delta = get_neuroncore_counter_value(entry, nc_id, ds_id); - nsysfsmetric_inc_counter(nd, NDS_NC_METRIC, ds_id, nc_id, delta, true); - } - - // read the custom sysfs metrics from nds. - nsysfsmetric_inc_counter(nd, NDS_NC_METRIC, NDS_NC_COUNTER_MODEL_LOAD_COUNT, nc_id, get_neuroncore_counter_value(entry, nc_id, NDS_NC_COUNTER_MODEL_LOAD_COUNT), true); - nsysfsmetric_inc_counter(nd, NDS_NC_METRIC, NDS_NC_COUNTER_INFERENCE_COUNT, nc_id, get_neuroncore_counter_value(entry, nc_id, NDS_NC_COUNTER_INFERENCE_COUNT), true); - nsysfsmetric_inc_counter(nd, NDS_NC_METRIC, NDS_NC_COUNTER_MAC_COUNT, nc_id, 2 * get_neuroncore_counter_value(entry, nc_id, NDS_NC_COUNTER_MAC_COUNT), true); // one MAC has two floating point operations (multiply and add) - nsysfsmetric_inc_counter(nd, NDS_NC_METRIC, NDS_NC_COUNTER_TIME_IN_USE, nc_id, get_neuroncore_counter_value(entry, nc_id, NDS_NC_COUNTER_TIME_IN_USE), true); - } -} diff --git a/neuron_sysfs_metrics.h b/neuron_sysfs_metrics.h index eb60681..97bd261 100644 --- a/neuron_sysfs_metrics.h +++ b/neuron_sysfs_metrics.h @@ -8,12 +8,13 @@ #define MAX_CHILD_NODES_NUM 32 #define NON_NDS_COUNTER_COUNT 64 -#define MAX_METRIC_ID (NDS_ND_COUNTER_COUNT + NDS_EXT_NC_COUNTER_LAST + NON_NDS_COUNTER_COUNT) +#define NDS_COUNTER_COUNT (NDS_EXT_NC_COUNTER_LAST + NDS_ND_COUNTER_COUNT) +#define MAX_METRIC_ID (NDS_COUNTER_COUNT + NON_NDS_COUNTER_COUNT) #define MAX_COUNTER_ATTR_TYPE_COUNT 3 #define NDS_NC_COUNTER_ID_TO_SYSFS_METRIC_ID(nds_id) (nds_id) #define NDS_ND_COUNTER_ID_TO_SYSFS_METRIC_ID(nds_id) (nds_id + NDS_EXT_NC_COUNTER_LAST) -#define NON_NDS_ID_TO_SYSFS_METRIC_ID(non_nds_id) (non_nds_id + NDS_ND_COUNTER_COUNT + NDS_EXT_NC_COUNTER_LAST) +#define NON_NDS_ID_TO_SYSFS_METRIC_ID(non_nds_id) (non_nds_id + NDS_COUNTER_COUNT) #define ATTR_INFO(_attr_name, _metric_id, _attr_type) { \ .attr_name = _attr_name, \ @@ -65,8 +66,10 @@ enum nsysfsmetric_non_nds_ids { // The metrics needed by sysfs metrics but not s NON_NDS_COUNTER_MODEL_LOAD_COUNT, NON_NDS_COUNTER_INFERENCE_COUNT, NON_NDS_COUNTER_ECC_SRAM_UNCORRECTED, + NON_NDS_COUNTER_ECC_SRAM_CORRECTED, NON_NDS_COUNTER_ECC_HBM_UNCORRECTED, NON_NDS_COUNTER_ECC_REPAIRABLE_HBM_UNCORRECTED, + NON_NDS_COUNTER_ECC_HBM_CORRECTED, NON_NDS_COUNTER_PE_ARRAY_ACTIVITY, NON_NDS_OTHER_NEURON_ARCH_TYPE, NON_NDS_OTHER_NEURON_INSTANCE_TYPE, @@ -74,10 +77,9 @@ enum nsysfsmetric_non_nds_ids { // The metrics needed by sysfs metrics but not s NON_NDS_OTHER_NOTIFY_DELAY, NON_NDS_OTHER_SERIAL_NUMBER, NON_NDS_OTHER_POWER_UTILIZATION, - NON_NDS_HEALTH_STATUS_SRAM_UE_COUNT, - NON_NDS_HEALTH_STATUS_HBM_UE_COUNT, - NON_NDS_HEALTH_STATUS_REPAIRABLE_HBM_UE_COUNT, NON_NDS_HEALTH_STATUS_HW_ERROR_EVENT, + NON_NDS_HEALTH_STATUS_OVERALL_HEALTH, + NON_NDS_HEALTH_STATUS_HEARTBEAT, NON_NDS_OTHER_POWER_UTILIZATION_RAW, }; @@ -92,13 +94,8 @@ struct sysfs_mem_thread { // Cache slot identifiers for misc RAM registers whose values are exposed under // stats/hardware/health_status/. Add a new value here when caching a new register. enum health_status_cache_slot { - HEALTH_STATUS_SLOT_SRAM_ECC, - HEALTH_STATUS_SLOT_HBM0_ECC, - HEALTH_STATUS_SLOT_HBM1_ECC, - HEALTH_STATUS_SLOT_HBM2_ECC, - HEALTH_STATUS_SLOT_HBM3_ECC, - HEALTH_STATUS_SLOT_HBM_REPAIR_STATE, - HEALTH_STATUS_SLOT_FW_API_VERSION, + HEALTH_STATUS_SLOT_HEALTH_STATUS_CHECK, + HEALTH_STATUS_SLOT_HEALTH_STATUS_SEQ, HEALTH_STATUS_SLOT_COUNT, }; @@ -119,6 +116,7 @@ struct nsysfsmetric_node { // represent a subdirectory in sysfs }; struct nsysfsmetric_metrics { // per neuron_device + struct neuron_device *nd; struct nsysfsmetric_node root; // represent the neuron device struct nsysfsmetric_node *dynamic_metrics_dirs[MAX_NC_PER_DEVICE]; struct nsysfsmetric_counter nrt_metrics[MAX_METRIC_ID][MAX_NC_PER_DEVICE]; // runtime metrics per NC, indiced by metric_id and nc_id @@ -132,6 +130,7 @@ struct nsysfsmetric_metrics { // per neuron_device // Indexed by enum health_status_cache_slot. u32 cached_health_regs[HEALTH_STATUS_SLOT_COUNT]; u32 hw_error_event_count; + bool health_check_regs_read_failed; struct nsysfsmetric_node *hardware_node; // stats/hardware/; cached so health_status can attach under it struct nsysfsmetric_node *health_status_node; // target for sysfs_notify on hw_error_event }; @@ -185,14 +184,6 @@ struct nsysfsmetric_node *nsysfsmetric_init_and_add_one_node(struct nsysfsmetric */ int nsysfsmetric_init_and_add_dynamic_counter_nodes(struct neuron_device *nd, uint64_t ds_val); -/** - * nsysfsmetric_nds_aggregate() - Aggregate sysfs metrics from datastore when a process exits - * - * @param nd: The pointer to the neuron_device - * @param entry: : The pointer to the datastore entry - */ -void nsysfsmetric_nds_aggregate(struct neuron_device *nd, struct neuron_datastore_entry *entry); - /** * nsysfsmetric_inc_counter() - Increment the counter with metric_id for neuron_device nd and neuron core nc_id by delta * diff --git a/share/neuron_driver_shared.h b/share/neuron_driver_shared.h index cbb73ad..cb6ba9d 100644 --- a/share/neuron_driver_shared.h +++ b/share/neuron_driver_shared.h @@ -23,7 +23,15 @@ enum neuron_driver_feature_flag { NEURON_DRIVER_FEATURE_ZEROCOPY = 1ull << 8, NEURON_DRIVER_FEATURE_PINNED_HOST_MEM = 1ull << 9, NEURON_DRIVER_FEATURE_ALLOC_WITH_PA = 1ull << 10, - NEURON_DRIVER_FEATURE_ASYNC_IO = 1ull << 11, + NEURON_DRIVER_FEATURE_ASYNC_RW = 1ull << 11, // async H2D/D2H read/write + NEURON_DRIVER_FEATURE_ASYNC_COPY = 1ull << 12, // async D2D copy +}; + +enum neuron_switch_fabric_bar_type { + NEURON_SWITCH_FABRIC_BAR_TYPE_HBM = 0x01, + NEURON_SWITCH_FABRIC_BAR_TYPE_CTRL = 0x02, + NEURON_SWITCH_FABRIC_BAR_TYPE_AC = 0x03, + NEURON_SWITCH_FABRIC_BAR_TYPE_MC = 0x04, }; // FIXME this should be more generic - like node type. @@ -361,6 +369,9 @@ enum { NDS_NC_COUNTER_COUNT = NDS_NC_COUNTER_LAST + NDS_NC_COUNTER_RESERVED }; +#define NDS_NC_MEM_USAGE_DEVICE_COUNTER_COUNT (NDS_NC_COUNTER_MEM_USAGE_MISC_DEVICE - NDS_NC_COUNTER_MEM_USAGE_CODE_DEVICE + 1) +#define NDS_NC_MEM_USAGE_DEVICE_SIZE (NDS_NC_MEM_USAGE_DEVICE_COUNTER_COUNT * sizeof(uint64_t)) + #define NDS_MAX_NEURONCORE_COUNT (4) #define NDS_EXT_MAX_NEURONCORE_COUNT (12) @@ -419,6 +430,11 @@ typedef struct nds_header { #define NDS_EXT_SECTION_SIZE_OLD (NDS_EXT_NEURONCORE_COUNTERS_SIZE_OLD + NDS_EXT_NEURONCORE_NC_DATA_SIZE_OLD) #define NDS_EXT_OFFSET_OLD (44588) +// Objects section: stores process info, model info map, model info entries, and process ext info. +// Starts immediately after the primary NC counters section and ends at the old extended counters offset. +#define NDS_OBJECTS_START (NDS_NEURONCORE_COUNTERS_START + NDS_NEURONCORE_COUNTERS_SIZE) +#define NDS_OBJECTS_SIZE (NDS_EXT_OFFSET_OLD - NDS_OBJECTS_START) + #define NDS_EXT_ALIGNMENT (64) #define NDS_ALIGN(v) ((v) + (-(v) & (NDS_EXT_ALIGNMENT - 1))) #define NDS_EXT_OFFSET (NDS_ALIGN(NDS_EXT_OFFSET_OLD + NDS_EXT_SECTION_SIZE_OLD)) diff --git a/v2/neuron_dhal_v2.c b/v2/neuron_dhal_v2.c index 3266ca0..592feb9 100644 --- a/v2/neuron_dhal_v2.c +++ b/v2/neuron_dhal_v2.c @@ -425,6 +425,11 @@ static u8 nnq_get_nqid_v2(struct neuron_device *nd, u8 nc_id, u8 index, u32 nq_t return (nq_type * V2_MAX_NQ_QUEUES) + index; } +static u8 nnq_get_nq_queue_count_v2(void) +{ + return V2_MAX_NQ_QUEUES; +} + /** * nnq_set_hwaddr() - set the physical address of the queue * @@ -1446,6 +1451,7 @@ int ndhal_register_funcs_v2(void) { ndhal->ndhal_nc.nc_get_event_addr = nc_get_event_addr_v2; ndhal->ndhal_nq.nnq_get_nqid = nnq_get_nqid_v2; ndhal->ndhal_nq.nnq_set_hwaddr = nnq_set_hwaddr_v2; + ndhal->ndhal_nq.nnq_get_nq_queue_count = nnq_get_nq_queue_count_v2; ndhal->ndhal_mpset.mp_min_alloc_size = (mempool_min_alloc_size < 1024) ? 1024 : mempool_min_alloc_size; // v2 has a bigger mem size and gen pool create fails if < 1024 ndhal->ndhal_mpset.small_pool_supported = true; ndhal->ndhal_mpset.mpset_set_dram_and_mpset_info = mpset_set_dram_and_mpset_info_v2; @@ -1470,6 +1476,7 @@ int ndhal_register_funcs_v2(void) { ndhal->ndhal_sysfs_metrics.nsysfsmetric_add_ecc_nodes = nsysfsmetric_add_ecc_nodes_v2; ndhal->ndhal_sysfs_metrics.nsysfsmetric_get_hbm_error_count = nsysfsmetric_get_hbm_error_count_v2; ndhal->ndhal_sysfs_metrics.nsysfsmetric_add_tensor_engine_node = nsysfsmetric_add_tensor_engine_node_v2; + ndhal->ndhal_sysfs_metrics.health_status_enabled = false; ndhal->ndhal_pci.axi_bar = BAR_UNUSED; ndhal->ndhal_pci.dram_bar = 4; ndhal->ndhal_pci.neuron_pci_get_device_id = neuron_pci_get_device_id_v2; diff --git a/v3/neuron_dhal_v3.c b/v3/neuron_dhal_v3.c index 678e84c..172c7ed 100644 --- a/v3/neuron_dhal_v3.c +++ b/v3/neuron_dhal_v3.c @@ -40,10 +40,6 @@ int force_die_flip = 0; module_param(force_die_flip, int, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); MODULE_PARM_DESC(force_die_flip, "Force Neuron Core Mapping APIs to give back DIE flip mappings"); -bool enable_sysfs_health_status_nodes = true; -module_param(enable_sysfs_health_status_nodes, bool, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); -MODULE_PARM_DESC(enable_sysfs_health_status_nodes, "Enable sysfs device health_status nodes"); - // TOP SP addresses are sparse on chip adjust to accommodate the table macro // #define V3_TOP_SP_GRP1_BASE V3_TOP_SP_0_BASE @@ -431,6 +427,7 @@ static int nr_post_reset_config_v3(struct neuron_device *nd, bool reset_successf if (nd->supports_hbm_7200 == -1) { ndhal->ndhal_perf.perf_update_hbm_7200_supported(nd); } + ndhal->ndhal_fw_io.fw_io_cache_bar_info(nd); } else { nd->supports_hbm_7200 = 0; } @@ -612,6 +609,11 @@ static u8 nnq_get_nqid_v3(struct neuron_device *nd, u8 nc_id, u8 index, u32 nq_t return (nq_type * V3_MAX_NQ_QUEUES) + index; } +static u8 nnq_get_nq_queue_count_v3(void) +{ + return V3_MAX_NQ_QUEUES; +} + /** * nnq_set_hwaddr() - set the physical address of the queue * @@ -1799,6 +1801,28 @@ static void perf_update_hbm_7200_supported_v3(struct neuron_device *nd) nd->supports_hbm_7200 = supports_hbm_7200; } +static void fw_io_cache_bar_info_v3(struct neuron_device *nd) +{ + int bar_idx; + int max_bars = 0; + + if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_MAX) { + max_bars = 2; + } else if (ndhal->ndhal_arch.platform_type == NEURON_PLATFORM_TYPE_PDS) { + max_bars = 1; + } + + for (bar_idx = 0; bar_idx < max_bars; bar_idx++) { + if (nd->bar_info[bar_idx].cached) { + continue; + } + fw_io_get_bar_info(nd->fw_io_ctx, bar_idx + 1, + nd->bar_info[bar_idx].bars, + &nd->bar_info[bar_idx].count); + nd->bar_info[bar_idx].cached = true; + } +} + /** * npe_class_node_id_show_data() - return sysfs class node_id * @@ -1953,6 +1977,7 @@ int ndhal_register_funcs_v3(void) { ndhal->ndhal_nc.nc_get_event_addr = nc_get_event_addr_v3; ndhal->ndhal_nq.nnq_get_nqid = nnq_get_nqid_v3; ndhal->ndhal_nq.nnq_set_hwaddr = nnq_set_hwaddr_v3; + ndhal->ndhal_nq.nnq_get_nq_queue_count = nnq_get_nq_queue_count_v3; ndhal->ndhal_mpset.mp_min_alloc_size = (mempool_min_alloc_size < 1024) ? 1024 : mempool_min_alloc_size; ndhal->ndhal_mpset.small_pool_supported = true; ndhal->ndhal_mpset.mpset_set_dram_and_mpset_info = mpset_set_dram_and_mpset_info_v3; @@ -1970,6 +1995,7 @@ int ndhal_register_funcs_v3(void) { ndhal->ndhal_fw_io.fw_io_read_csr_array = fw_io_read_csr_array_v3; ndhal->ndhal_fw_io.fw_io_execute_request = fw_io_execute_request_v3; ndhal->ndhal_fw_io.fw_io_post_metric = fw_io_post_metric_v3; + ndhal->ndhal_fw_io.fw_io_cache_bar_info = fw_io_cache_bar_info_v3; ndhal->ndhal_mmap.dm_mmap_special = dm_mmap_special_v3; ndhal->ndhal_mmap.mmap_get_bar4_offset = mmap_get_bar4_offset_v3; ndhal->ndhal_sysfs_metrics.root_info_node_attrs_info_tbl_cnt = root_info_node_attrs_info_tbl_cnt_v3; @@ -1977,7 +2003,7 @@ int ndhal_register_funcs_v3(void) { ndhal->ndhal_sysfs_metrics.nsysfsmetric_add_ecc_nodes = nsysfsmetric_add_ecc_nodes_v3; ndhal->ndhal_sysfs_metrics.nsysfsmetric_get_hbm_error_count = nsysfsmetric_get_hbm_error_count_v3; ndhal->ndhal_sysfs_metrics.nsysfsmetric_add_tensor_engine_node = nsysfsmetric_add_tensor_engine_node_v3; - ndhal->ndhal_sysfs_metrics.health_status_enabled = enable_sysfs_health_status_nodes; + ndhal->ndhal_sysfs_metrics.health_status_enabled = true; ndhal->ndhal_pci.axi_bar = BAR_UNUSED; ndhal->ndhal_pci.apb_bar = 0; ndhal->ndhal_pci.dram_bar = 4; diff --git a/v4/neuron_dhal_v4.c b/v4/neuron_dhal_v4.c index 6dadfde..cb656da 100644 --- a/v4/neuron_dhal_v4.c +++ b/v4/neuron_dhal_v4.c @@ -13,6 +13,7 @@ #include "../neuron_reset.h" #include "../neuron_arch.h" #include "../neuron_cdev.h" +#include "../neuron_dma.h" #include "../neuron_pci.h" #include "../v3/neuron_pelect.h" @@ -150,11 +151,14 @@ static int ndhal_register_funcs_trn3(void) { static const struct neuron_platform_lookup v4_platform_map[] = { {"trn3e.24xlarge", NEURON_PLATFORM_TYPE_MAX}, {"trn3e-dev0.24xlarge", NEURON_PLATFORM_TYPE_MAX}, - {"trn3.48xlarge", NEURON_PLATFORM_TYPE_PDS}, - {"trn3-dev0.48xlarge", NEURON_PLATFORM_TYPE_PDS}, - {"trn3-dev1.48xlarge", NEURON_PLATFORM_TYPE_PDS}, - {"trn3p.48xlarge", NEURON_PLATFORM_TYPE_ULTRASERVER}, - {NULL, NEURON_PLATFORM_TYPE_INVALID}, + {"trn3.48xlarge", NEURON_PLATFORM_TYPE_PDS}, + {"trn3-dev0.48xlarge", NEURON_PLATFORM_TYPE_PDS}, + {"trn3-dev1.48xlarge", NEURON_PLATFORM_TYPE_PDS}, + {"trn3s-es.48xlarge", NEURON_PLATFORM_TYPE_PDS}, + {"trn3sn.48xlarge", NEURON_PLATFORM_TYPE_PDS}, + {"trn3sn-es.48xlarge", NEURON_PLATFORM_TYPE_PDS}, + {"trn3p.48xlarge", NEURON_PLATFORM_TYPE_ULTRASERVER}, + {NULL, NEURON_PLATFORM_TYPE_INVALID}, }; static enum neuron_platform_type ndhal_platform_type_v4(void) @@ -171,12 +175,22 @@ static enum neuron_platform_type ndhal_platform_type_v4(void) static bool ndhal_instance_type_3xl(void) { + static const char *const trn3_3xl_instance_names[] = { + "trn3pd98.3xlarge", + "trn3s-es.3xlarge", + "trn3sn.3xlarge", + "trn3sn-es.3xlarge", + }; static bool instance_type_is_3xl = false; -#define NEURON_TRN3PD98_3XL_INSTANCE_NAME "trn3pd98.3xlarge" char buf[128]; + int i; + if (narch_get_instance_type_name(buf, sizeof(buf))) goto done; - if (strncmp(buf, NEURON_TRN3PD98_3XL_INSTANCE_NAME, sizeof(NEURON_TRN3PD98_3XL_INSTANCE_NAME)-1) == 0) { - instance_type_is_3xl = true; + for (i = 0; i < sizeof(trn3_3xl_instance_names) / sizeof(*trn3_3xl_instance_names); i++) { + if (strncmp(buf, trn3_3xl_instance_names[i], strlen(trn3_3xl_instance_names[i])) == 0) { + instance_type_is_3xl = true; + break; + } } done: @@ -184,29 +198,27 @@ static bool ndhal_instance_type_3xl(void) } /** - * ndmar_ctx_queue_bit_v4() - dummy ctx queue bitmap mapping for v4 - * @h2d_eng_id: ignored - * @qid: ignored + * ndmar_ctx_queue_bit_v4() - map a V4 H2D queue to a dense bitmap index + * @h2d_eng_id: DMA engine id in the V4 H2D/D2H engine range (128 to 131) + * @qid: DMA queue id within the engine * - * Async IO is not supported on v4 yet, so this hook is unused. + * Return bitmap bit index for the queue. */ static int ndmar_ctx_queue_bit_v4(uint32_t h2d_eng_id, uint32_t qid) { - return 0; + return (h2d_eng_id - V4_D2H_0_IDX) * DMA_MAX_Q_V4 + qid; } /** - * ndmar_ctx_queue_from_bit_v4() - dummy ctx queue bitmap reverse mapping for v4 - * @bit: ignored - * @h2d_eng_id: returned DMA engine id placeholder - * @qid: returned DMA queue id placeholder - * - * Async IO is not supported on v4 yet, so this hook is unused. + * ndmar_ctx_queue_from_bit_v4() - map a dense bitmap index back to a V4 H2D queue + * @bit: bitmap bit index + * @h2d_eng_id: returned DMA engine id + * @qid: returned DMA queue id */ static void ndmar_ctx_queue_from_bit_v4(int bit, uint32_t *h2d_eng_id, uint32_t *qid) { - *h2d_eng_id = 0; - *qid = 0; + *h2d_eng_id = V4_D2H_0_IDX + (bit / DMA_MAX_Q_V4); + *qid = bit % DMA_MAX_Q_V4; } @@ -495,6 +507,7 @@ int ndhal_register_funcs_v4(void) { ndhal->ndhal_ndmar.ndmar_ctx_queue_from_bit = ndmar_ctx_queue_from_bit_v4; ndhal->ndhal_cdev.ncdev_mem_regions = ncdev_mem_regions_v4; ndhal->ndhal_perf.perf_update_hbm_7200_supported = perf_update_hbm_7200_supported_v4; + ndhal->ndhal_sysfs_metrics.health_status_enabled = true; if (narch_is_emu()) { // Temporarily disable resets on emulation until support is ready