AI/rocm-systems
2
0
Garpu 0
Kode Masalah Tarik Permintaan Actions Packages Projects Rilis Wiki Kegiatan

[SWDEV-243250] RDC Process Start/Stop integration (#189)

Telusuri Sumber 
Change-Id: I3d2be33b5d23cd259b3d06fb572f81d19e6c3798

Signed-off-by: adapryor <Adam.pryor@amd.com>
This commit is contained in:
Pryor, Adam
2025-06-02 14:42:21 -05:00
committed by GitHub
orang tua ae6b1aa6e6
melakukan 0e9c3b2c4f
13 mengubah file dengan 242 tambahan dan 27 penghapusan
Download Patch File Download Diff File Expand all files Collapse all files
rdc_libs/rdc/src/RdcCacheManagerImpl.cc
+45 -1
Melihat File
@@ -196,6 +196,40 @@ rdc_status_t RdcCacheManagerImpl::rdc_update_job_stats(uint32_t gpu_index,
return RDC_ST_NOT_FOUND;
}
auto& je = cache_jobs_.at(job_id);
// handle process‐start/stop
if (value.field_id == RDC_EVNT_NOTIF_PROCESS_START ||
value.field_id == RDC_EVNT_NOTIF_PROCESS_END) {
rdc_process_status_info_t info{};
sscanf(value.value.str, "PID: %u task: %255s", &info.pid, info.process_name);
uint64_t ts_us = value.ts * 1000;
if (value.field_id == RDC_EVNT_NOTIF_PROCESS_START) {
info.start_time = ts_us;
} else {
info.stop_time = ts_us;
}
auto pit = je.pid_to_index.find(info.pid);
if (pit == je.pid_to_index.end()) {
uint32_t slot = je.num_processes;
if (slot < RDC_MAX_NUM_PROCESSES_STATUS) {
je.processes[slot] = info;
je.pid_to_index[info.pid] = slot;
je.num_processes = slot + 1;
}
} else {
auto& entry = je.processes[pit->second];
if (value.field_id == RDC_EVNT_NOTIF_PROCESS_START) {
entry.start_time = info.start_time;
} else {
entry.stop_time = info.stop_time;
}
}
return RDC_ST_OK;
}
auto gpu_iter = job_iter->second.gpu_stats.find(gpu_index);
if (gpu_iter == job_iter->second.gpu_stats.end()) {
return RDC_ST_NOT_FOUND;
@@ -380,6 +414,13 @@ rdc_status_t RdcCacheManagerImpl::rdc_job_get_stats(const char jobId[64],
set_average_summary(summary_info.gpu_temperature, p_job_info->num_gpus);
set_average_summary(summary_info.memory_clock, p_job_info->num_gpus);
// Set process start/stop info
auto& je = cache_jobs_[jobId];
p_job_info->num_processes = je.num_processes;
for (uint32_t i = 0; i < je.num_processes; ++i) {
p_job_info->processes[i] = je.processes[i];
}
return RDC_ST_OK;
}
@@ -392,9 +433,12 @@ rdc_status_t RdcCacheManagerImpl::rdc_job_start_stats(const char job_id[64],
const rdc_group_info_t& ginfo,
const rdc_field_group_info_t& finfo,
const rdc_gpu_gauges_t& gpu_gauges) {
RdcJobStatsCacheEntry cacheEntry;
RdcJobStatsCacheEntry cacheEntry{};
cacheEntry.start_time = std::time(nullptr);
cacheEntry.end_time = 0;
cacheEntry.num_processes = 0;
cacheEntry.pid_to_index.clear();
for (auto& p : cacheEntry.processes) p = {};
for (unsigned int i = 0; i < ginfo.count; i++) { // GPUs
GpuSummaryStats gstats;
gstats.energy_consumed = 0;
rdc_libs/rdc/src/RdcNotificationImpl.cc
+24 -5
Melihat File
@@ -45,7 +45,15 @@ static std::unordered_map<rdc_field_t, amdsmi_evt_notification_type_t> rdc_2_smi
{RDC_EVNT_NOTIF_THERMAL_THROTTLE, AMDSMI_EVT_NOTIF_THERMAL_THROTTLE},
{RDC_EVNT_NOTIF_PRE_RESET, AMDSMI_EVT_NOTIF_GPU_PRE_RESET},
{RDC_EVNT_NOTIF_POST_RESET, AMDSMI_EVT_NOTIF_GPU_POST_RESET},
{RDC_EVNT_NOTIF_RING_HANG, AMDSMI_EVT_NOTIF_RING_HANG},
{RDC_EVNT_NOTIF_MIGRATE_START, AMDSMI_EVT_NOTIF_MIGRATE_START},
{RDC_EVNT_NOTIF_MIGRATE_END, AMDSMI_EVT_NOTIF_MIGRATE_END},
{RDC_EVNT_NOTIF_PAGE_FAULT_START, AMDSMI_EVT_NOTIF_PAGE_FAULT_START},
{RDC_EVNT_NOTIF_PAGE_FAULT_END, AMDSMI_EVT_NOTIF_PAGE_FAULT_END},
{RDC_EVNT_NOTIF_QUEUE_EVICTION, AMDSMI_EVT_NOTIF_QUEUE_EVICTION},
{RDC_EVNT_NOTIF_QUEUE_RESTORE, AMDSMI_EVT_NOTIF_QUEUE_RESTORE},
{RDC_EVNT_NOTIF_UNMAP_FROM_GPU, AMDSMI_EVT_NOTIF_UNMAP_FROM_GPU},
{RDC_EVNT_NOTIF_PROCESS_START, AMDSMI_EVT_NOTIF_PROCESS_START},
{RDC_EVNT_NOTIF_PROCESS_END, AMDSMI_EVT_NOTIF_PROCESS_END},
};
static std::unordered_map<amdsmi_evt_notification_type_t, rdc_field_t> smi_event_notif_2_rdc_map = {
{AMDSMI_EVT_NOTIF_VMFAULT, RDC_EVNT_NOTIF_VMFAULT},
@@ -53,7 +61,15 @@ static std::unordered_map<amdsmi_evt_notification_type_t, rdc_field_t> smi_event
{AMDSMI_EVT_NOTIF_THERMAL_THROTTLE, RDC_EVNT_NOTIF_THERMAL_THROTTLE},
{AMDSMI_EVT_NOTIF_GPU_PRE_RESET, RDC_EVNT_NOTIF_PRE_RESET},
{AMDSMI_EVT_NOTIF_GPU_POST_RESET, RDC_EVNT_NOTIF_POST_RESET},
{AMDSMI_EVT_NOTIF_RING_HANG, RDC_EVNT_NOTIF_RING_HANG},
{AMDSMI_EVT_NOTIF_MIGRATE_START, RDC_EVNT_NOTIF_MIGRATE_START},
{AMDSMI_EVT_NOTIF_MIGRATE_END, RDC_EVNT_NOTIF_MIGRATE_END},
{AMDSMI_EVT_NOTIF_PAGE_FAULT_START, RDC_EVNT_NOTIF_PAGE_FAULT_START},
{AMDSMI_EVT_NOTIF_PAGE_FAULT_END, RDC_EVNT_NOTIF_PAGE_FAULT_END},
{AMDSMI_EVT_NOTIF_QUEUE_EVICTION, RDC_EVNT_NOTIF_QUEUE_EVICTION},
{AMDSMI_EVT_NOTIF_QUEUE_RESTORE, RDC_EVNT_NOTIF_QUEUE_RESTORE},
{AMDSMI_EVT_NOTIF_UNMAP_FROM_GPU, RDC_EVNT_NOTIF_UNMAP_FROM_GPU},
{AMDSMI_EVT_NOTIF_PROCESS_START, RDC_EVNT_NOTIF_PROCESS_START},
{AMDSMI_EVT_NOTIF_PROCESS_END, RDC_EVNT_NOTIF_PROCESS_END},
};
// This const determines space allocated on stack for notification events.
@@ -160,9 +176,12 @@ rdc_status_t RdcNotificationImpl::listen(rdc_evnt_notification_t* events, uint32
for (uint32_t i = 0; i < f_cnt; ++i) {
assert(smi_event_notif_2_rdc_map.find(smi_events[i].event) != smi_event_notif_2_rdc_map.end());
uint64_t bdfid;
amdsmi_get_gpu_bdf_id(smi_events[i].processor_handle, &bdfid);
events[i].gpu_id = bdfid;
uint32_t gpu_id;
ret = get_gpu_id_from_processor_handle(smi_events[i].processor_handle, &gpu_id);
if (ret != AMDSMI_STATUS_SUCCESS) {
return Smi2RdcError(ret);
}
events[i].gpu_id = gpu_id;
events[i].field.field_id = smi_event_notif_2_rdc_map[smi_events[i].event];
events[i].field.status = RDC_ST_OK;
events[i].field.ts = now;
rdc_libs/rdc/src/RdcWatchTableImpl.cc
+13 -5
Melihat File
@@ -73,6 +73,19 @@ rdc_status_t RdcWatchTableImpl::rdc_job_start_stats(rdc_gpu_group_t group_id, co
return RDC_ST_NOT_FOUND;
}
// Add process start/stop for all jobs
rdc_group_info_t ginfo;
result = group_settings_->rdc_group_gpu_get_info(group_id, &ginfo);
if (result != RDC_ST_OK) {
return result;
}
for (uint32_t ix = 0; ix < ginfo.count; ++ix) {
uint32_t gpu = ginfo.entity_ids[ix];
fields_in_watch.emplace_back(gpu, RDC_EVNT_NOTIF_PROCESS_START);
fields_in_watch.emplace_back(gpu, RDC_EVNT_NOTIF_PROCESS_END);
}
JobWatchTableEntry jentry{group_id, fields_in_watch};
do { //< lock guard for thread safe
std::lock_guard<std::mutex> guard(watch_mutex_);
@@ -80,11 +93,6 @@ rdc_status_t RdcWatchTableImpl::rdc_job_start_stats(rdc_gpu_group_t group_id, co
} while (0);
rdc_field_group_info_t finfo;
rdc_group_info_t ginfo;
result = group_settings_->rdc_group_gpu_get_info(group_id, &ginfo);
if (result != RDC_ST_OK) {
return result;
}
result = group_settings_->rdc_group_field_get_info(JOB_FIELD_ID, &finfo);
if (result != RDC_ST_OK) {
rdc_libs/rdc/src/SmiUtils.cc
+31
Melihat File
@@ -136,6 +136,37 @@ amdsmi_status_t get_processor_handle_from_id(uint32_t gpu_id,
return AMDSMI_STATUS_SUCCESS;
}
amdsmi_status_t get_gpu_id_from_processor_handle(amdsmi_processor_handle processor_handle,
uint32_t* gpu_index) {
if (!gpu_index) {
return AMDSMI_STATUS_INVAL;
}
std::vector<amdsmi_socket_handle> sockets;
auto ret = get_socket_handles(sockets);
if (ret != AMDSMI_STATUS_SUCCESS) {
return ret;
}
uint32_t idx = 0;
for (auto const& sock : sockets) {
std::vector<amdsmi_processor_handle> procs;
ret = get_processor_handles(sock, procs);
if (ret != AMDSMI_STATUS_SUCCESS) {
return ret;
}
for (auto const& h : procs) {
if (h == processor_handle) {
*gpu_index = idx;
return AMDSMI_STATUS_SUCCESS;
}
++idx;
}
}
return AMDSMI_STATUS_INPUT_OUT_OF_BOUNDS;
}
amdsmi_status_t get_processor_count(uint32_t& all_processor_count) {
uint32_t total_processor_count = 0;
uint32_t socket_count;
rdc_libs/rdc_client/src/RdcStandaloneHandler.cc
+12
Melihat File
@@ -170,6 +170,18 @@ rdc_status_t RdcStandaloneHandler::rdc_job_get_stats(const char job_id[64],
copy_gpu_usage_info(reply.gpus(i), &(p_job_info->gpus[i]));
}
p_job_info->num_processes = reply.num_processes();
for (int i = 0; i < reply.num_processes(); i++) {
const auto& proc_msg = reply.processes(i);
p_job_info->processes[i].pid = proc_msg.pid();
strncpy_with_null(p_job_info->processes[i].process_name, proc_msg.process_name().c_str(),
MAX_PROCESS_NAME - 1);
p_job_info->processes[i].start_time = proc_msg.start_time();
p_job_info->processes[i].stop_time = proc_msg.stop_time();
}
return RDC_ST_OK;
}