AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[SWDEV-243250] RDC Process Start/Stop integration (#189)

Browse Source 
Change-Id: I3d2be33b5d23cd259b3d06fb572f81d19e6c3798

Signed-off-by: adapryor <Adam.pryor@amd.com>

[ROCm/rdc commit: 0e9c3b2c4f]
This commit is contained in:
Pryor, Adam
2025-06-02 14:42:21 -05:00
committed by GitHub
parent 30397e77f3
commit ec661d5d17
13 changed files with 242 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rdc/include/rdc/rdc.h
+25 -2
View File
@@ -399,9 +399,17 @@ typedef enum {
//!< due to temperature rise
RDC_EVNT_NOTIF_PRE_RESET, //!< GPU reset is about to occur
RDC_EVNT_NOTIF_POST_RESET, //!< GPU reset just occurred
RDC_EVNT_NOTIF_RING_HANG, //!< GPU ring hang just occurred
RDC_EVNT_NOTIF_MIGRATE_START,
RDC_EVNT_NOTIF_MIGRATE_END,
RDC_EVNT_NOTIF_PAGE_FAULT_START,
RDC_EVNT_NOTIF_PAGE_FAULT_END,
RDC_EVNT_NOTIF_QUEUE_EVICTION,
RDC_EVNT_NOTIF_QUEUE_RESTORE,
RDC_EVNT_NOTIF_UNMAP_FROM_GPU,
RDC_EVNT_NOTIF_PROCESS_START,
RDC_EVNT_NOTIF_PROCESS_END,
RDC_EVNT_NOTIF_LAST = RDC_EVNT_NOTIF_RING_HANG,
RDC_EVNT_NOTIF_LAST = RDC_EVNT_NOTIF_PROCESS_END,
/**
* @brief RDC health related fields
@@ -496,6 +504,18 @@ typedef struct {
rdc_stats_summary_t memory_utilization; //!< Memory Utilization statistics
} rdc_gpu_usage_info_t; //!< GPU usage statistics
#define MAX_PROCESS_NAME 256
/**
* @brief The structure to track process start/stop times during a job running
*/
typedef struct {
uint32_t pid; //!< Process ID
char process_name[MAX_PROCESS_NAME];
uint64_t start_time; //!< Process start time in microseconds since 1970
uint64_t stop_time; //!< Process stop time in microseconds since 1970
} rdc_process_status_info_t;
#define RDC_MAX_NUM_PROCESSES_STATUS 64
/**
* @brief The structure to hold the job stats
*/
@@ -504,6 +524,9 @@ typedef struct {
rdc_gpu_usage_info_t summary; //!< Job usage summary statistics
//!< (overall)
rdc_gpu_usage_info_t gpus[16]; //!< Job usage summary statistics by GPU
uint32_t num_processes; //!< Number of processes tracked
rdc_process_status_info_t
processes[RDC_MAX_NUM_PROCESSES_STATUS]; //!< Array to track process start/stop times
} rdc_job_info_t;
/**
projects/rdc/include/rdc_lib/impl/RdcCacheManagerImpl.h
+9 -10
View File
@@ -78,6 +78,10 @@ struct RdcJobStatsCacheEntry {
uint64_t start_time;
uint64_t end_time;
std::map<uint32_t, GpuSummaryStats> gpu_stats;
uint32_t num_processes = 0;
std::array<rdc_process_status_info_t, RDC_MAX_NUM_PROCESSES_STATUS> processes{};
std::map<uint32_t, uint32_t> pid_to_index;
};
// <job_id, job_stats>
@@ -110,19 +114,14 @@ class RdcCacheManagerImpl : public RdcCacheManager {
rdc_status_t rdc_job_remove(const char job_id[64]) override;
rdc_status_t rdc_job_remove_all() override;
rdc_status_t rdc_health_set(rdc_gpu_group_t group_id,
uint32_t gpu_index,
rdc_status_t rdc_health_set(rdc_gpu_group_t group_id, uint32_t gpu_index,
const rdc_field_value& value) override;
rdc_status_t rdc_health_get_values(rdc_gpu_group_t group_id,
uint32_t gpu_index,
rdc_field_t field_id,
uint64_t start_timestamp,
uint64_t end_timestamp,
rdc_field_value* start_value,
rdc_status_t rdc_health_get_values(rdc_gpu_group_t group_id, uint32_t gpu_index,
rdc_field_t field_id, uint64_t start_timestamp,
uint64_t end_timestamp, rdc_field_value* start_value,
rdc_field_value* end_value) override;
rdc_status_t rdc_health_clear(rdc_gpu_group_t group_id) override;
rdc_status_t rdc_update_health_stats(rdc_gpu_group_t group_id,
uint32_t gpu_index,
rdc_status_t rdc_update_health_stats(rdc_gpu_group_t group_id, uint32_t gpu_index,
const rdc_field_value& value) override;
private:
projects/rdc/include/rdc_lib/impl/SmiUtils.h
+2
View File
@@ -34,6 +34,8 @@ namespace rdc {
rdc_status_t Smi2RdcError(amdsmi_status_t rsmi);
amdsmi_status_t get_processor_handle_from_id(uint32_t gpu_id,
amdsmi_processor_handle* processor_handle);
amdsmi_status_t get_gpu_id_from_processor_handle(amdsmi_processor_handle processor_handle,
uint32_t* gpu_index);
amdsmi_status_t get_processor_count(uint32_t& all_processor_count);
amdsmi_status_t get_socket_handles(std::vector<amdsmi_socket_handle>& sockets);
amdsmi_status_t get_processor_handles(amdsmi_socket_handle socket,