AI/rocm-systems
2
0
Rozštěpit 0
Zdrojový kód Úkoly Pull requesty Akce Balíčky Projekty Vydání Wiki Aktivita

[RDC] [SWDEV-551280] RDC to include Error Counters (#1087)

Procházet zdrojové kódy 
* rdc error counter

* RDC error counters

* fix

* Updates

* updated field names

Signed-off-by: yalmusaf_amdeng <yalmusaf@amd.com>

---------

Signed-off-by: yalmusaf_amdeng <yalmusaf@amd.com>
Co-authored-by: yalmusaf_amdeng <yalmusaf@amd.com>
Tento commit je obsažen v:
Yazen AL Musaffar
2025-12-03 15:22:18 -06:00
odevzdal GitHub
rodič c9d6a8720c
revize 16b9160034
8 změnil soubory, kde provedl 95 přidání a 3 odebrání
Stáhněte soubor záplaty Stáhněte rozdílový soubor Rozbalit všechny soubory Sbalit všechny soubory
projects/rdc/common/rdc_field.data
+3
Zobrazit soubor
@@ -55,6 +55,9 @@ FLD_DESC_ENT(RDC_FI_POWER_USAGE, "Power usage in microwatts",
FLD_DESC_ENT(RDC_FI_PCIE_TX, "PCIe Tx utilization in bytes/second", "PCIE_TX", true)
FLD_DESC_ENT(RDC_FI_PCIE_RX, "PCIe Rx utilization in bytes/second", "PCIE_RX", true)
FLD_DESC_ENT(RDC_FI_PCIE_BANDWIDTH, "PCIe bandwidth in Mbps", "PCIE_BANDWIDTH", true)
FLD_DESC_ENT(RDC_FI_PCIE_LC_PERF_OTHER_END_RECOVERY, "PCIe end recovery counter", "PCIE_RECOVERY_COUNT", true)
FLD_DESC_ENT(RDC_FI_PCIE_NAK_RCVD_COUNT_ACC, "PCIe nak received counter", "PCIE_NAK_RCVD_COUNT", true)
FLD_DESC_ENT(RDC_FI_PCIE_NAK_SENT_COUNT_ACC, "PCIe nak sent counter", "PCIE_NAK_SENT_COUNT", true)
FLD_DESC_ENT(RDC_FI_GPU_UTIL, "GPU busy percentage", "GPU_UTIL", true)
FLD_DESC_ENT(RDC_FI_GPU_MEMORY_USAGE, "Memory usage of the GPU instance in bytes", "GPU_MEMORY_USAGE", true)
projects/rdc/include/rdc/rdc.h
+6
Zobrazit soubor
@@ -193,6 +193,9 @@ typedef enum {
// RDC_FI_PCIE_TX, RDC_FI_PCIE_RX are not supported on new ASIC
// The RDC_FI_PCIE_BANDWIDTH should be used
RDC_FI_PCIE_BANDWIDTH, //!< PCIe bandwidth in Mbps
RDC_FI_PCIE_LC_PERF_OTHER_END_RECOVERY, //!< PCIe link recovery count
RDC_FI_PCIE_NAK_RCVD_COUNT_ACC, //!< PCIe NAK received count
RDC_FI_PCIE_NAK_SENT_COUNT_ACC, //!< PCIe NAK sent count
/**
* @brief GPU usage related fields
@@ -516,6 +519,9 @@ typedef struct {
rdc_stats_summary_t pcie_total; //!< Total PCIe bandwidth stats
//!< pcie_tx/pcie_rx are not available on mi300, max integer
//!< returned, so use pcie_total
uint32_t pcie_lc_perf_other_end_recovery_count; //!< PCIE other end recovery count
uint32_t pcie_nak_sent_count_acc; //!< PCIE NAK sent accumulated count
uint32_t pcie_nak_rcvd_count_acc; //!< PCIE NAK received accumulated count
rdc_stats_summary_t power_usage; //!< GPU Power usage stats
rdc_stats_summary_t gpu_clock; //!< GPU Clock speed stats
rdc_stats_summary_t memory_clock; //!< Mem. Clock speed stats
projects/rdc/include/rdc_lib/impl/RdcCacheManagerImpl.h
+3
Zobrazit soubor
@@ -71,6 +71,9 @@ struct GpuSummaryStats {
uint64_t energy_last_time;
uint64_t ecc_correct_init; // Init counter when job starts
uint64_t ecc_uncorrect_init; // Init counter when job starts
uint32_t pcie_lc_perf_other_end_recovery_count_init; // Init counter when job starts
uint32_t pcie_nak_sent_count_acc_init; // Init counter when job starts
uint32_t pcie_nak_rcvd_count_acc_init; // Init counter when job starts
std::map<uint32_t, FieldSummaryStats> field_summaries;
};
projects/rdc/python_binding/rdc_bootstrap.py
+3
Zobrazit soubor
@@ -95,6 +95,9 @@ class rdc_field_t(c_int):
RDC_FI_PCIE_TX = 400
RDC_FI_PCIE_RX = 401
RDC_FI_PCIE_BANDWIDTH = 402
RDC_FI_PCIE_LC_PERF_OTHER_END_RECOVERY = 403
RDC_FI_PCIE_NAK_RCVD_COUNT_ACC = 404
RDC_FI_PCIE_NAK_SENT_COUNT_ACC = 405
RDC_FI_GPU_UTIL = 500
RDC_FI_GPU_MEMORY_USAGE = 501
RDC_FI_GPU_MEMORY_TOTAL = 502
projects/rdc/rdc_libs/rdc/src/RdcCacheManagerImpl.cc
+69 -1
Zobrazit soubor
@@ -330,6 +330,9 @@ rdc_status_t RdcCacheManagerImpl::rdc_job_get_stats(const char jobId[64],
summary_info.pcie_tx = {0, std::numeric_limits<uint64_t>::max(), 0, 0};
summary_info.pcie_rx = {0, std::numeric_limits<uint64_t>::max(), 0, 0};
summary_info.pcie_total = {0, std::numeric_limits<uint64_t>::max(), 0, 0};
summary_info.pcie_lc_perf_other_end_recovery_count = 0;
summary_info.pcie_nak_sent_count_acc = 0;
summary_info.pcie_nak_rcvd_count_acc = 0;
summary_info.gpu_temperature = {0, std::numeric_limits<uint64_t>::max(), 0, 0};
summary_info.memory_clock = {0, std::numeric_limits<uint64_t>::max(), 0, 0};
summary_info.gpu_clock = {0, std::numeric_limits<uint64_t>::max(), 0, 0};
@@ -371,6 +374,39 @@ rdc_status_t RdcCacheManagerImpl::rdc_job_get_stats(const char jobId[64],
gpu_info.ecc_uncorrect = 0;
}
if (is_job_stopped) {
gpu_info.pcie_lc_perf_other_end_recovery_count = gpus->second.pcie_lc_perf_other_end_recovery_count_init;
summary_info.pcie_lc_perf_other_end_recovery_count += gpu_info.pcie_lc_perf_other_end_recovery_count;
} else if (gpu_gauges.find({gpus->first, RDC_FI_PCIE_LC_PERF_OTHER_END_RECOVERY}) != gpu_gauges.end()) {
gpu_info.pcie_lc_perf_other_end_recovery_count = gpu_gauges.at({gpus->first, RDC_FI_PCIE_LC_PERF_OTHER_END_RECOVERY}) -
gpus->second.pcie_lc_perf_other_end_recovery_count_init;
summary_info.pcie_lc_perf_other_end_recovery_count += gpu_info.pcie_lc_perf_other_end_recovery_count;
} else {
gpu_info.pcie_lc_perf_other_end_recovery_count = 0;
}
if (is_job_stopped) {
gpu_info.pcie_nak_sent_count_acc = gpus->second.pcie_nak_sent_count_acc_init;
summary_info.pcie_nak_sent_count_acc += gpu_info.pcie_nak_sent_count_acc;
} else if (gpu_gauges.find({gpus->first, RDC_FI_PCIE_NAK_SENT_COUNT_ACC}) != gpu_gauges.end()) {
gpu_info.pcie_nak_sent_count_acc = gpu_gauges.at({gpus->first, RDC_FI_PCIE_NAK_SENT_COUNT_ACC}) -
gpus->second.pcie_nak_sent_count_acc_init;
summary_info.pcie_nak_sent_count_acc += gpu_info.pcie_nak_sent_count_acc;
} else {
gpu_info.pcie_nak_sent_count_acc = 0;
}
if (is_job_stopped) {
gpu_info.pcie_nak_rcvd_count_acc = gpus->second.pcie_nak_rcvd_count_acc_init;
summary_info.pcie_nak_rcvd_count_acc += gpu_info.pcie_nak_rcvd_count_acc;
} else if (gpu_gauges.find({gpus->first, RDC_FI_PCIE_NAK_RCVD_COUNT_ACC}) != gpu_gauges.end()) {
gpu_info.pcie_nak_rcvd_count_acc = gpu_gauges.at({gpus->first, RDC_FI_PCIE_NAK_RCVD_COUNT_ACC}) -
gpus->second.pcie_nak_rcvd_count_acc_init;
summary_info.pcie_nak_rcvd_count_acc += gpu_info.pcie_nak_rcvd_count_acc;
} else {
gpu_info.pcie_nak_rcvd_count_acc = 0;
}
if (gpu_gauges.find({gpus->first, RDC_FI_GPU_MEMORY_TOTAL}) == gpu_gauges.end()) {
RDC_LOG(RDC_ERROR, "Cannot find the total memory");
return RDC_ST_BAD_PARAMETER;
@@ -402,7 +438,6 @@ rdc_status_t RdcCacheManagerImpl::rdc_job_get_stats(const char jobId[64],
} else if (ite->first == RDC_FI_PCIE_BANDWIDTH) {
set_summary(ite->second, gpu_info.pcie_total, summary_info.pcie_total, 1);
}
}
}
// Set the average of the summary
set_average_summary(summary_info.power_usage, p_job_info->num_gpus);
@@ -424,6 +459,8 @@ rdc_status_t RdcCacheManagerImpl::rdc_job_get_stats(const char jobId[64],
return RDC_ST_OK;
}
}
void RdcCacheManagerImpl::set_average_summary(rdc_stats_summary_t& summary, uint32_t num_gpus) {
summary.average = summary.average / num_gpus;
summary.standard_deviation = summary.standard_deviation / num_gpus;
@@ -460,6 +497,22 @@ rdc_status_t RdcCacheManagerImpl::rdc_job_start_stats(const char job_id[64],
gstats.ecc_uncorrect_init = gpu_gauges.at({ginfo.entity_ids[i], RDC_FI_ECC_UNCORRECT_TOTAL});
}
gstats.pcie_lc_perf_other_end_recovery_count_init = 0;
if (gpu_gauges.find({ginfo.entity_ids[i], RDC_FI_PCIE_LC_PERF_OTHER_END_RECOVERY}) != gpu_gauges.end()) {
gstats.pcie_lc_perf_other_end_recovery_count_init = gpu_gauges.at({ginfo.entity_ids[i], RDC_FI_PCIE_LC_PERF_OTHER_END_RECOVERY});
}
gstats.pcie_nak_sent_count_acc_init = 0;
if (gpu_gauges.find({ginfo.entity_ids[i], RDC_FI_PCIE_NAK_SENT_COUNT_ACC}) != gpu_gauges.end()) {
gstats.pcie_nak_sent_count_acc_init = gpu_gauges.at({ginfo.entity_ids[i], RDC_FI_PCIE_NAK_SENT_COUNT_ACC});
}
gstats.pcie_nak_rcvd_count_acc_init = 0;
if (gpu_gauges.find({ginfo.entity_ids[i], RDC_FI_PCIE_NAK_RCVD_COUNT_ACC}) != gpu_gauges.end()) {
gstats.pcie_nak_rcvd_count_acc_init = gpu_gauges.at({ginfo.entity_ids[i], RDC_FI_PCIE_NAK_RCVD_COUNT_ACC});
}
cacheEntry.gpu_stats.insert({ginfo.entity_ids[i], gstats});
}
@@ -493,6 +546,21 @@ rdc_status_t RdcCacheManagerImpl::rdc_job_stop_stats(const char job_id[64],
gpus->second.ecc_uncorrect_init = gpu_gauges.at({gpus->first, RDC_FI_ECC_UNCORRECT_TOTAL}) -
gpus->second.ecc_uncorrect_init;
}
if (gpu_gauges.find({gpus->first, RDC_FI_PCIE_LC_PERF_OTHER_END_RECOVERY}) != gpu_gauges.end()) {
gpus->second.pcie_lc_perf_other_end_recovery_count_init = gpu_gauges.at({gpus->first, RDC_FI_PCIE_LC_PERF_OTHER_END_RECOVERY}) -
gpus->second.pcie_lc_perf_other_end_recovery_count_init;
}
if (gpu_gauges.find({gpus->first, RDC_FI_PCIE_NAK_SENT_COUNT_ACC}) != gpu_gauges.end()) {
gpus->second.pcie_nak_sent_count_acc_init = gpu_gauges.at({gpus->first, RDC_FI_PCIE_NAK_SENT_COUNT_ACC}) -
gpus->second.pcie_nak_sent_count_acc_init;
}
if (gpu_gauges.find({gpus->first, RDC_FI_PCIE_NAK_RCVD_COUNT_ACC}) != gpu_gauges.end()) {
gpus->second.pcie_nak_rcvd_count_acc_init = gpu_gauges.at({gpus->first, RDC_FI_PCIE_NAK_RCVD_COUNT_ACC}) -
gpus->second.pcie_nak_rcvd_count_acc_init;
}
}
return RDC_ST_OK;
projects/rdc/rdc_libs/rdc/src/RdcGroupSettingsImpl.cc
+3 -1
Zobrazit soubor
@@ -37,7 +37,9 @@ RdcGroupSettingsImpl::RdcGroupSettingsImpl(const RdcPartitionPtr& partition)
// Add the default job stats fields
rdc_field_t job_fields[] = {RDC_FI_GPU_MEMORY_USAGE, RDC_FI_POWER_USAGE, RDC_FI_GPU_CLOCK,
RDC_FI_GPU_UTIL, RDC_FI_PCIE_TX, RDC_FI_PCIE_RX,
RDC_FI_PCIE_BANDWIDTH, RDC_FI_MEM_CLOCK, RDC_FI_GPU_TEMP};
RDC_FI_PCIE_BANDWIDTH, RDC_FI_PCIE_LC_PERF_OTHER_END_RECOVERY,
RDC_FI_PCIE_NAK_RCVD_COUNT_ACC, RDC_FI_PCIE_NAK_SENT_COUNT_ACC,
RDC_FI_MEM_CLOCK, RDC_FI_GPU_TEMP};
char job_field_group[] = "JobStatsFields";
rdc_field_grp_t fgid = JOB_FIELD_ID;
projects/rdc/rdc_libs/rdc/src/RdcMetricFetcherImpl.cc
+6
Zobrazit soubor
@@ -522,6 +522,9 @@ rdc_status_t RdcMetricFetcherImpl::fetch_gpu_field_(uint32_t gpu_index, rdc_fiel
{RDC_FI_XGMI_7_WRITE_KB, gpu_metrics.xgmi_write_data_acc[7]},
{RDC_FI_XGMI_TOTAL_WRITE_KB, sum_xgmi_write(gpu_metrics)},
{RDC_FI_PCIE_BANDWIDTH, gpu_metrics.pcie_bandwidth_inst},
{RDC_FI_PCIE_LC_PERF_OTHER_END_RECOVERY, gpu_metrics.pcie_lc_perf_other_end_recovery},
{RDC_FI_PCIE_NAK_RCVD_COUNT_ACC, gpu_metrics.pcie_nak_rcvd_count_acc},
{RDC_FI_PCIE_NAK_SENT_COUNT_ACC, gpu_metrics.pcie_nak_sent_count_acc},
};
// In gpu_metrics,the max value means not supported
@@ -899,6 +902,9 @@ rdc_status_t RdcMetricFetcherImpl::fetch_gpu_field_(uint32_t gpu_index, rdc_fiel
case RDC_FI_XGMI_7_WRITE_KB:
case RDC_FI_XGMI_TOTAL_WRITE_KB:
case RDC_FI_PCIE_BANDWIDTH:
case RDC_FI_PCIE_LC_PERF_OTHER_END_RECOVERY:
case RDC_FI_PCIE_NAK_RCVD_COUNT_ACC:
case RDC_FI_PCIE_NAK_SENT_COUNT_ACC:
read_gpu_metrics_uint64_t();
break;
case RDC_HEALTH_XGMI_ERROR: {
projects/rdc/rdc_libs/rdc/src/RdcSmiLib.cc
+2 -1
Zobrazit soubor
@@ -153,7 +153,8 @@ rdc_status_t RdcSmiLib::rdc_telemetry_fields_query(uint32_t field_ids[MAX_NUM_FI
RDC_FI_GPU_COUNT, RDC_FI_DEV_NAME, RDC_FI_GPU_CLOCK,
RDC_FI_MEM_CLOCK, RDC_FI_MEMORY_TEMP, RDC_FI_GPU_TEMP,
RDC_FI_POWER_USAGE, RDC_FI_PCIE_TX, RDC_FI_PCIE_RX,
RDC_FI_PCIE_BANDWIDTH, RDC_FI_GPU_UTIL, RDC_FI_GPU_MEMORY_USAGE,
RDC_FI_PCIE_BANDWIDTH, RDC_FI_PCIE_LC_PERF_OTHER_END_RECOVERY, RDC_FI_PCIE_NAK_RCVD_COUNT_ACC,
RDC_FI_PCIE_NAK_SENT_COUNT_ACC, RDC_FI_GPU_UTIL, RDC_FI_GPU_MEMORY_USAGE,
RDC_FI_GPU_MEMORY_TOTAL, RDC_FI_ECC_CORRECT_TOTAL, RDC_FI_ECC_UNCORRECT_TOTAL,
RDC_FI_ECC_SDMA_CE, RDC_FI_ECC_SDMA_UE, RDC_FI_ECC_GFX_CE,
RDC_FI_ECC_GFX_UE, RDC_FI_ECC_MMHUB_CE, RDC_FI_ECC_MMHUB_UE,