From 6fb53cdb1b9ad3ba8d4a9ed314dd1211e95c503c Mon Sep 17 00:00:00 2001 From: Alex Sierra Date: Thu, 3 Nov 2022 21:13:33 +0000 Subject: [PATCH] src: add debug API to support GPU core dump Functions to API added to extract the following information from KFD Runtime information, device info and queues snapshot. Signed-off-by: Alex Sierra Change-Id: If995ecc54497ab61189bb0f209c64af0bbb0f56f [ROCm/ROCR-Runtime commit: 0cbf26c148c87bd69af9be2949132f17c0b6c1a2] --- projects/rocr-runtime/include/hsakmt.h | 40 ++++++ projects/rocr-runtime/src/debug.c | 161 ++++++++++++++++++++++++ projects/rocr-runtime/src/libhsakmt.ver | 4 + 3 files changed, 205 insertions(+) diff --git a/projects/rocr-runtime/include/hsakmt.h b/projects/rocr-runtime/include/hsakmt.h index dae3f5a980..c95afb8a07 100644 --- a/projects/rocr-runtime/include/hsakmt.h +++ b/projects/rocr-runtime/include/hsakmt.h @@ -734,6 +734,46 @@ hsaKmtGetRuntimeCapabilities( HSAuint32 *caps_mask // OUT ); +/** + Enable debug trap. +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtDbgEnable( + void **runtime_info, //Out + HSAuint32 *data_size //Out + ); + +/** + Disable debug trap. +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtDbgDisable(void); + +/** + Get device snapshot. +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtDbgGetDeviceData( + void **data, //Out + HSAuint32 *n_entries, //Out + HSAuint32 *entry_size //Out + ); + +/** + Get queues snapshot. +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtDbgGetQueueData( + void **data, //Out + HSAuint32 *n_entries, //Out + HSAuint32 *entry_size, //Out + bool suspend_queues //In + ); + /** Gets GPU and CPU clock counters for particular Node */ diff --git a/projects/rocr-runtime/src/debug.c b/projects/rocr-runtime/src/debug.c index 51022a2ba3..e97b809844 100644 --- a/projects/rocr-runtime/src/debug.c +++ b/projects/rocr-runtime/src/debug.c @@ -351,3 +351,164 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetRuntimeCapabilities(HSAuint32 *caps_mask) *caps_mask = runtime_capabilities_mask; return HSAKMT_STATUS_SUCCESS; } + +static HSAKMT_STATUS dbg_trap_get_device_data(void *data, + uint32_t *n_entries, + uint32_t entry_size) +{ + struct kfd_ioctl_dbg_trap_args args = {0}; + + args.device_snapshot.snapshot_buf_ptr = (uint64_t) data; + args.device_snapshot.num_devices = *n_entries; + args.device_snapshot.entry_size = entry_size; + args.op = KFD_IOC_DBG_TRAP_GET_DEVICE_SNAPSHOT; + args.pid = getpid(); + if (kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, &args)) + return HSAKMT_STATUS_ERROR; + *n_entries = args.device_snapshot.num_devices; + + return HSAKMT_STATUS_SUCCESS; +} + +static HSAKMT_STATUS dbg_trap_get_queue_data(void *data, + uint32_t *n_entries, + uint32_t entry_size, + uint32_t *queue_ids) +{ + struct kfd_ioctl_dbg_trap_args args = {0}; + + args.queue_snapshot.num_queues = *n_entries; + args.queue_snapshot.entry_size = entry_size; + args.queue_snapshot.exception_mask = KFD_EC_MASK(EC_QUEUE_NEW); + args.op = KFD_IOC_DBG_TRAP_GET_QUEUE_SNAPSHOT; + args.queue_snapshot.snapshot_buf_ptr = (uint64_t) data; + args.pid = getpid(); + + if (kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, &args)) + return HSAKMT_STATUS_ERROR; + + *n_entries = args.queue_snapshot.num_queues; + if (queue_ids && *n_entries) { + struct kfd_queue_snapshot_entry *queue_entry = + (struct kfd_queue_snapshot_entry *) data; + for (uint32_t i = 0; i < *n_entries; i++) + queue_ids[i] = queue_entry[i].queue_id; + } + + return HSAKMT_STATUS_SUCCESS; +} + +static HSAKMT_STATUS dbg_trap_suspend_queues(uint32_t *queue_ids, + uint32_t num_queues) +{ + struct kfd_ioctl_dbg_trap_args args = {0}; + int r; + + args.suspend_queues.queue_array_ptr = (uint64_t) queue_ids; + args.suspend_queues.num_queues = num_queues; + args.suspend_queues.exception_mask = KFD_EC_MASK(EC_QUEUE_NEW); + args.op = KFD_IOC_DBG_TRAP_SUSPEND_QUEUES; + args.pid = getpid(); + + r = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, &args); + if (r < 0) + return HSAKMT_STATUS_ERROR; + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtDbgEnable(void **runtime_info, + HSAuint32 *data_size) +{ + struct kfd_ioctl_dbg_trap_args args = {0}; + + CHECK_KFD_OPEN(); + CHECK_KFD_MINOR_VERSION(KFD_IOCTL_MINOR_VERSION); + *data_size = sizeof(struct kfd_runtime_info); + args.enable.rinfo_size = *data_size; + args.enable.dbg_fd = kfd_fd; + *runtime_info = malloc(args.enable.rinfo_size); + if (!*runtime_info) + return HSAKMT_STATUS_NO_MEMORY; + args.enable.rinfo_ptr = (uint64_t) *runtime_info; + args.op = KFD_IOC_DBG_TRAP_ENABLE; + args.pid = getpid(); + + if (kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, &args)) { + free(*runtime_info); + return HSAKMT_STATUS_ERROR; + } + + return HSAKMT_STATUS_SUCCESS; +} +HSAKMT_STATUS HSAKMTAPI hsaKmtDbgDisable(void) +{ + struct kfd_ioctl_dbg_trap_args args = {0}; + + CHECK_KFD_OPEN(); + CHECK_KFD_MINOR_VERSION(KFD_IOCTL_MINOR_VERSION); + args.enable.dbg_fd = kfd_fd; + args.op = KFD_IOC_DBG_TRAP_DISABLE; + args.pid = getpid(); + + if (kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, &args)) + return HSAKMT_STATUS_ERROR; + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtDbgGetDeviceData(void **data, + HSAuint32 *n_entries, + HSAuint32 *entry_size) +{ + HSAKMT_STATUS ret = HSAKMT_STATUS_NO_MEMORY; + + CHECK_KFD_OPEN(); + CHECK_KFD_MINOR_VERSION(KFD_IOCTL_MINOR_VERSION); + *n_entries = UINT32_MAX; + *entry_size = sizeof(struct kfd_dbg_device_info_entry); + *data = malloc(*entry_size * *n_entries); + if (!*data) + return ret; + ret = dbg_trap_get_device_data(*data, n_entries, *entry_size); + if (ret) + free(*data); + + return ret; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtDbgGetQueueData(void **data, + HSAuint32 *n_entries, + HSAuint32 *entry_size, + bool suspend_queues) +{ + uint32_t *queue_ids = NULL; + + CHECK_KFD_OPEN(); + CHECK_KFD_MINOR_VERSION(KFD_IOCTL_MINOR_VERSION); + *entry_size = sizeof(struct kfd_queue_snapshot_entry); + *n_entries = 0; + if (dbg_trap_get_queue_data(NULL, n_entries, *entry_size, NULL)) + return HSAKMT_STATUS_ERROR; + *data = malloc(*n_entries * *entry_size); + if (!*data) + return HSAKMT_STATUS_NO_MEMORY; + if (suspend_queues && *n_entries) + queue_ids = (uint32_t *)malloc(sizeof(uint32_t) * *n_entries); + if (!queue_ids || + dbg_trap_get_queue_data(*data, n_entries, *entry_size, queue_ids)) + goto free_data; + if (queue_ids) { + if (dbg_trap_suspend_queues(queue_ids, *n_entries) || + dbg_trap_get_queue_data(*data, n_entries, *entry_size, NULL)) + goto free_data; + free(queue_ids); + } + return HSAKMT_STATUS_SUCCESS; +free_data: + free(*data); + if (queue_ids) + free(queue_ids); + + return HSAKMT_STATUS_ERROR; +} diff --git a/projects/rocr-runtime/src/libhsakmt.ver b/projects/rocr-runtime/src/libhsakmt.ver index 80aedc224f..dc17215263 100644 --- a/projects/rocr-runtime/src/libhsakmt.ver +++ b/projects/rocr-runtime/src/libhsakmt.ver @@ -42,6 +42,10 @@ hsaKmtDbgRegister; hsaKmtDbgUnregister; hsaKmtDbgWavefrontControl; hsaKmtDbgAddressWatch; +hsaKmtDbgEnable; +hsaKmtDbgDisable; +hsaKmtDbgGetDeviceData; +hsaKmtDbgGetQueueData; hsaKmtGetClockCounters; hsaKmtPmcGetCounterProperties; hsaKmtPmcRegisterTrace;