diff --git a/projects/rdc/common/rdc_field.data b/projects/rdc/common/rdc_field.data index 0bc3350868..bdb904d658 100644 --- a/projects/rdc/common/rdc_field.data +++ b/projects/rdc/common/rdc_field.data @@ -55,6 +55,9 @@ FLD_DESC_ENT(RDC_FI_POWER_USAGE, "Power usage in microwatts", FLD_DESC_ENT(RDC_FI_PCIE_TX, "PCIe Tx utilization in bytes/second", "PCIE_TX", true) FLD_DESC_ENT(RDC_FI_PCIE_RX, "PCIe Rx utilization in bytes/second", "PCIE_RX", true) FLD_DESC_ENT(RDC_FI_PCIE_BANDWIDTH, "PCIe bandwidth in Mbps", "PCIE_BANDWIDTH", true) +FLD_DESC_ENT(RDC_FI_PCIE_LC_PERF_OTHER_END_RECOVERY, "PCIe end recovery counter", "PCIE_RECOVERY_COUNT", true) +FLD_DESC_ENT(RDC_FI_PCIE_NAK_RCVD_COUNT_ACC, "PCIe nak received counter", "PCIE_NAK_RCVD_COUNT", true) +FLD_DESC_ENT(RDC_FI_PCIE_NAK_SENT_COUNT_ACC, "PCIe nak sent counter", "PCIE_NAK_SENT_COUNT", true) FLD_DESC_ENT(RDC_FI_GPU_UTIL, "GPU busy percentage", "GPU_UTIL", true) FLD_DESC_ENT(RDC_FI_GPU_MEMORY_USAGE, "Memory usage of the GPU instance in bytes", "GPU_MEMORY_USAGE", true) diff --git a/projects/rdc/include/rdc/rdc.h b/projects/rdc/include/rdc/rdc.h index 33b730add2..1d9b904d36 100644 --- a/projects/rdc/include/rdc/rdc.h +++ b/projects/rdc/include/rdc/rdc.h @@ -193,6 +193,9 @@ typedef enum { // RDC_FI_PCIE_TX, RDC_FI_PCIE_RX are not supported on new ASIC // The RDC_FI_PCIE_BANDWIDTH should be used RDC_FI_PCIE_BANDWIDTH, //!< PCIe bandwidth in Mbps + RDC_FI_PCIE_LC_PERF_OTHER_END_RECOVERY, //!< PCIe link recovery count + RDC_FI_PCIE_NAK_RCVD_COUNT_ACC, //!< PCIe NAK received count + RDC_FI_PCIE_NAK_SENT_COUNT_ACC, //!< PCIe NAK sent count /** * @brief GPU usage related fields @@ -516,6 +519,9 @@ typedef struct { rdc_stats_summary_t pcie_total; //!< Total PCIe bandwidth stats //!< pcie_tx/pcie_rx are not available on mi300, max integer //!< returned, so use pcie_total + uint32_t pcie_lc_perf_other_end_recovery_count; //!< PCIE other end recovery count + uint32_t pcie_nak_sent_count_acc; //!< PCIE NAK sent accumulated count + uint32_t pcie_nak_rcvd_count_acc; //!< PCIE NAK received accumulated count rdc_stats_summary_t power_usage; //!< GPU Power usage stats rdc_stats_summary_t gpu_clock; //!< GPU Clock speed stats rdc_stats_summary_t memory_clock; //!< Mem. Clock speed stats diff --git a/projects/rdc/include/rdc_lib/impl/RdcCacheManagerImpl.h b/projects/rdc/include/rdc_lib/impl/RdcCacheManagerImpl.h index 4baa863946..45c8e132a4 100644 --- a/projects/rdc/include/rdc_lib/impl/RdcCacheManagerImpl.h +++ b/projects/rdc/include/rdc_lib/impl/RdcCacheManagerImpl.h @@ -71,6 +71,9 @@ struct GpuSummaryStats { uint64_t energy_last_time; uint64_t ecc_correct_init; // Init counter when job starts uint64_t ecc_uncorrect_init; // Init counter when job starts + uint32_t pcie_lc_perf_other_end_recovery_count_init; // Init counter when job starts + uint32_t pcie_nak_sent_count_acc_init; // Init counter when job starts + uint32_t pcie_nak_rcvd_count_acc_init; // Init counter when job starts std::map field_summaries; }; diff --git a/projects/rdc/python_binding/rdc_bootstrap.py b/projects/rdc/python_binding/rdc_bootstrap.py index c707f93cce..ad50b733c3 100644 --- a/projects/rdc/python_binding/rdc_bootstrap.py +++ b/projects/rdc/python_binding/rdc_bootstrap.py @@ -95,6 +95,9 @@ class rdc_field_t(c_int): RDC_FI_PCIE_TX = 400 RDC_FI_PCIE_RX = 401 RDC_FI_PCIE_BANDWIDTH = 402 + RDC_FI_PCIE_LC_PERF_OTHER_END_RECOVERY = 403 + RDC_FI_PCIE_NAK_RCVD_COUNT_ACC = 404 + RDC_FI_PCIE_NAK_SENT_COUNT_ACC = 405 RDC_FI_GPU_UTIL = 500 RDC_FI_GPU_MEMORY_USAGE = 501 RDC_FI_GPU_MEMORY_TOTAL = 502 diff --git a/projects/rdc/rdc_libs/rdc/src/RdcCacheManagerImpl.cc b/projects/rdc/rdc_libs/rdc/src/RdcCacheManagerImpl.cc index f91004d618..3f6010bfa0 100644 --- a/projects/rdc/rdc_libs/rdc/src/RdcCacheManagerImpl.cc +++ b/projects/rdc/rdc_libs/rdc/src/RdcCacheManagerImpl.cc @@ -330,6 +330,9 @@ rdc_status_t RdcCacheManagerImpl::rdc_job_get_stats(const char jobId[64], summary_info.pcie_tx = {0, std::numeric_limits::max(), 0, 0}; summary_info.pcie_rx = {0, std::numeric_limits::max(), 0, 0}; summary_info.pcie_total = {0, std::numeric_limits::max(), 0, 0}; + summary_info.pcie_lc_perf_other_end_recovery_count = 0; + summary_info.pcie_nak_sent_count_acc = 0; + summary_info.pcie_nak_rcvd_count_acc = 0; summary_info.gpu_temperature = {0, std::numeric_limits::max(), 0, 0}; summary_info.memory_clock = {0, std::numeric_limits::max(), 0, 0}; summary_info.gpu_clock = {0, std::numeric_limits::max(), 0, 0}; @@ -371,6 +374,39 @@ rdc_status_t RdcCacheManagerImpl::rdc_job_get_stats(const char jobId[64], gpu_info.ecc_uncorrect = 0; } + if (is_job_stopped) { + gpu_info.pcie_lc_perf_other_end_recovery_count = gpus->second.pcie_lc_perf_other_end_recovery_count_init; + summary_info.pcie_lc_perf_other_end_recovery_count += gpu_info.pcie_lc_perf_other_end_recovery_count; + } else if (gpu_gauges.find({gpus->first, RDC_FI_PCIE_LC_PERF_OTHER_END_RECOVERY}) != gpu_gauges.end()) { + gpu_info.pcie_lc_perf_other_end_recovery_count = gpu_gauges.at({gpus->first, RDC_FI_PCIE_LC_PERF_OTHER_END_RECOVERY}) - + gpus->second.pcie_lc_perf_other_end_recovery_count_init; + summary_info.pcie_lc_perf_other_end_recovery_count += gpu_info.pcie_lc_perf_other_end_recovery_count; + } else { + gpu_info.pcie_lc_perf_other_end_recovery_count = 0; + } + + if (is_job_stopped) { + gpu_info.pcie_nak_sent_count_acc = gpus->second.pcie_nak_sent_count_acc_init; + summary_info.pcie_nak_sent_count_acc += gpu_info.pcie_nak_sent_count_acc; + } else if (gpu_gauges.find({gpus->first, RDC_FI_PCIE_NAK_SENT_COUNT_ACC}) != gpu_gauges.end()) { + gpu_info.pcie_nak_sent_count_acc = gpu_gauges.at({gpus->first, RDC_FI_PCIE_NAK_SENT_COUNT_ACC}) - + gpus->second.pcie_nak_sent_count_acc_init; + summary_info.pcie_nak_sent_count_acc += gpu_info.pcie_nak_sent_count_acc; + } else { + gpu_info.pcie_nak_sent_count_acc = 0; + } + + if (is_job_stopped) { + gpu_info.pcie_nak_rcvd_count_acc = gpus->second.pcie_nak_rcvd_count_acc_init; + summary_info.pcie_nak_rcvd_count_acc += gpu_info.pcie_nak_rcvd_count_acc; + } else if (gpu_gauges.find({gpus->first, RDC_FI_PCIE_NAK_RCVD_COUNT_ACC}) != gpu_gauges.end()) { + gpu_info.pcie_nak_rcvd_count_acc = gpu_gauges.at({gpus->first, RDC_FI_PCIE_NAK_RCVD_COUNT_ACC}) - + gpus->second.pcie_nak_rcvd_count_acc_init; + summary_info.pcie_nak_rcvd_count_acc += gpu_info.pcie_nak_rcvd_count_acc; + } else { + gpu_info.pcie_nak_rcvd_count_acc = 0; + } + if (gpu_gauges.find({gpus->first, RDC_FI_GPU_MEMORY_TOTAL}) == gpu_gauges.end()) { RDC_LOG(RDC_ERROR, "Cannot find the total memory"); return RDC_ST_BAD_PARAMETER; @@ -402,7 +438,6 @@ rdc_status_t RdcCacheManagerImpl::rdc_job_get_stats(const char jobId[64], } else if (ite->first == RDC_FI_PCIE_BANDWIDTH) { set_summary(ite->second, gpu_info.pcie_total, summary_info.pcie_total, 1); } - } } // Set the average of the summary set_average_summary(summary_info.power_usage, p_job_info->num_gpus); @@ -424,6 +459,8 @@ rdc_status_t RdcCacheManagerImpl::rdc_job_get_stats(const char jobId[64], return RDC_ST_OK; } +} + void RdcCacheManagerImpl::set_average_summary(rdc_stats_summary_t& summary, uint32_t num_gpus) { summary.average = summary.average / num_gpus; summary.standard_deviation = summary.standard_deviation / num_gpus; @@ -460,6 +497,22 @@ rdc_status_t RdcCacheManagerImpl::rdc_job_start_stats(const char job_id[64], gstats.ecc_uncorrect_init = gpu_gauges.at({ginfo.entity_ids[i], RDC_FI_ECC_UNCORRECT_TOTAL}); } + gstats.pcie_lc_perf_other_end_recovery_count_init = 0; + if (gpu_gauges.find({ginfo.entity_ids[i], RDC_FI_PCIE_LC_PERF_OTHER_END_RECOVERY}) != gpu_gauges.end()) { + gstats.pcie_lc_perf_other_end_recovery_count_init = gpu_gauges.at({ginfo.entity_ids[i], RDC_FI_PCIE_LC_PERF_OTHER_END_RECOVERY}); + } + + gstats.pcie_nak_sent_count_acc_init = 0; + if (gpu_gauges.find({ginfo.entity_ids[i], RDC_FI_PCIE_NAK_SENT_COUNT_ACC}) != gpu_gauges.end()) { + gstats.pcie_nak_sent_count_acc_init = gpu_gauges.at({ginfo.entity_ids[i], RDC_FI_PCIE_NAK_SENT_COUNT_ACC}); + } + + gstats.pcie_nak_rcvd_count_acc_init = 0; + if (gpu_gauges.find({ginfo.entity_ids[i], RDC_FI_PCIE_NAK_RCVD_COUNT_ACC}) != gpu_gauges.end()) { + gstats.pcie_nak_rcvd_count_acc_init = gpu_gauges.at({ginfo.entity_ids[i], RDC_FI_PCIE_NAK_RCVD_COUNT_ACC}); + } + + cacheEntry.gpu_stats.insert({ginfo.entity_ids[i], gstats}); } @@ -493,6 +546,21 @@ rdc_status_t RdcCacheManagerImpl::rdc_job_stop_stats(const char job_id[64], gpus->second.ecc_uncorrect_init = gpu_gauges.at({gpus->first, RDC_FI_ECC_UNCORRECT_TOTAL}) - gpus->second.ecc_uncorrect_init; } + + if (gpu_gauges.find({gpus->first, RDC_FI_PCIE_LC_PERF_OTHER_END_RECOVERY}) != gpu_gauges.end()) { + gpus->second.pcie_lc_perf_other_end_recovery_count_init = gpu_gauges.at({gpus->first, RDC_FI_PCIE_LC_PERF_OTHER_END_RECOVERY}) - + gpus->second.pcie_lc_perf_other_end_recovery_count_init; + } + + if (gpu_gauges.find({gpus->first, RDC_FI_PCIE_NAK_SENT_COUNT_ACC}) != gpu_gauges.end()) { + gpus->second.pcie_nak_sent_count_acc_init = gpu_gauges.at({gpus->first, RDC_FI_PCIE_NAK_SENT_COUNT_ACC}) - + gpus->second.pcie_nak_sent_count_acc_init; + } + + if (gpu_gauges.find({gpus->first, RDC_FI_PCIE_NAK_RCVD_COUNT_ACC}) != gpu_gauges.end()) { + gpus->second.pcie_nak_rcvd_count_acc_init = gpu_gauges.at({gpus->first, RDC_FI_PCIE_NAK_RCVD_COUNT_ACC}) - + gpus->second.pcie_nak_rcvd_count_acc_init; + } } return RDC_ST_OK; diff --git a/projects/rdc/rdc_libs/rdc/src/RdcGroupSettingsImpl.cc b/projects/rdc/rdc_libs/rdc/src/RdcGroupSettingsImpl.cc index 5aa897218c..c120584099 100644 --- a/projects/rdc/rdc_libs/rdc/src/RdcGroupSettingsImpl.cc +++ b/projects/rdc/rdc_libs/rdc/src/RdcGroupSettingsImpl.cc @@ -37,7 +37,9 @@ RdcGroupSettingsImpl::RdcGroupSettingsImpl(const RdcPartitionPtr& partition) // Add the default job stats fields rdc_field_t job_fields[] = {RDC_FI_GPU_MEMORY_USAGE, RDC_FI_POWER_USAGE, RDC_FI_GPU_CLOCK, RDC_FI_GPU_UTIL, RDC_FI_PCIE_TX, RDC_FI_PCIE_RX, - RDC_FI_PCIE_BANDWIDTH, RDC_FI_MEM_CLOCK, RDC_FI_GPU_TEMP}; + RDC_FI_PCIE_BANDWIDTH, RDC_FI_PCIE_LC_PERF_OTHER_END_RECOVERY, + RDC_FI_PCIE_NAK_RCVD_COUNT_ACC, RDC_FI_PCIE_NAK_SENT_COUNT_ACC, + RDC_FI_MEM_CLOCK, RDC_FI_GPU_TEMP}; char job_field_group[] = "JobStatsFields"; rdc_field_grp_t fgid = JOB_FIELD_ID; diff --git a/projects/rdc/rdc_libs/rdc/src/RdcMetricFetcherImpl.cc b/projects/rdc/rdc_libs/rdc/src/RdcMetricFetcherImpl.cc index 04669929de..b0623b7eaf 100644 --- a/projects/rdc/rdc_libs/rdc/src/RdcMetricFetcherImpl.cc +++ b/projects/rdc/rdc_libs/rdc/src/RdcMetricFetcherImpl.cc @@ -522,6 +522,9 @@ rdc_status_t RdcMetricFetcherImpl::fetch_gpu_field_(uint32_t gpu_index, rdc_fiel {RDC_FI_XGMI_7_WRITE_KB, gpu_metrics.xgmi_write_data_acc[7]}, {RDC_FI_XGMI_TOTAL_WRITE_KB, sum_xgmi_write(gpu_metrics)}, {RDC_FI_PCIE_BANDWIDTH, gpu_metrics.pcie_bandwidth_inst}, + {RDC_FI_PCIE_LC_PERF_OTHER_END_RECOVERY, gpu_metrics.pcie_lc_perf_other_end_recovery}, + {RDC_FI_PCIE_NAK_RCVD_COUNT_ACC, gpu_metrics.pcie_nak_rcvd_count_acc}, + {RDC_FI_PCIE_NAK_SENT_COUNT_ACC, gpu_metrics.pcie_nak_sent_count_acc}, }; // In gpu_metrics,the max value means not supported @@ -899,6 +902,9 @@ rdc_status_t RdcMetricFetcherImpl::fetch_gpu_field_(uint32_t gpu_index, rdc_fiel case RDC_FI_XGMI_7_WRITE_KB: case RDC_FI_XGMI_TOTAL_WRITE_KB: case RDC_FI_PCIE_BANDWIDTH: + case RDC_FI_PCIE_LC_PERF_OTHER_END_RECOVERY: + case RDC_FI_PCIE_NAK_RCVD_COUNT_ACC: + case RDC_FI_PCIE_NAK_SENT_COUNT_ACC: read_gpu_metrics_uint64_t(); break; case RDC_HEALTH_XGMI_ERROR: { diff --git a/projects/rdc/rdc_libs/rdc/src/RdcSmiLib.cc b/projects/rdc/rdc_libs/rdc/src/RdcSmiLib.cc index e89ddf2d50..080a2e4b73 100644 --- a/projects/rdc/rdc_libs/rdc/src/RdcSmiLib.cc +++ b/projects/rdc/rdc_libs/rdc/src/RdcSmiLib.cc @@ -153,7 +153,8 @@ rdc_status_t RdcSmiLib::rdc_telemetry_fields_query(uint32_t field_ids[MAX_NUM_FI RDC_FI_GPU_COUNT, RDC_FI_DEV_NAME, RDC_FI_GPU_CLOCK, RDC_FI_MEM_CLOCK, RDC_FI_MEMORY_TEMP, RDC_FI_GPU_TEMP, RDC_FI_POWER_USAGE, RDC_FI_PCIE_TX, RDC_FI_PCIE_RX, - RDC_FI_PCIE_BANDWIDTH, RDC_FI_GPU_UTIL, RDC_FI_GPU_MEMORY_USAGE, + RDC_FI_PCIE_BANDWIDTH, RDC_FI_PCIE_LC_PERF_OTHER_END_RECOVERY, RDC_FI_PCIE_NAK_RCVD_COUNT_ACC, + RDC_FI_PCIE_NAK_SENT_COUNT_ACC, RDC_FI_GPU_UTIL, RDC_FI_GPU_MEMORY_USAGE, RDC_FI_GPU_MEMORY_TOTAL, RDC_FI_ECC_CORRECT_TOTAL, RDC_FI_ECC_UNCORRECT_TOTAL, RDC_FI_ECC_SDMA_CE, RDC_FI_ECC_SDMA_UE, RDC_FI_ECC_GFX_CE, RDC_FI_ECC_GFX_UE, RDC_FI_ECC_MMHUB_CE, RDC_FI_ECC_MMHUB_UE,