From b10500a77029c555717caf28e9bb038ef8dc43e6 Mon Sep 17 00:00:00 2001 From: Chris Freehill Date: Thu, 30 Apr 2020 17:07:39 -0500 Subject: [PATCH] Add event notification API Change-Id: Ib6e8efbe6cdefaa7de1f74bd26993e9b4b011649 [ROCm/amdsmi commit: 2235ede34c456f1c7d3490f6fe74825d442d272e] --- projects/amdsmi/CMakeLists.txt | 2 + projects/amdsmi/include/rocm_smi/kfd_ioctl.h | 567 ++++++++++++++++++ projects/amdsmi/include/rocm_smi/rocm_smi.h | 131 ++++ .../amdsmi/include/rocm_smi/rocm_smi_device.h | 8 + .../amdsmi/include/rocm_smi/rocm_smi_main.h | 11 + projects/amdsmi/src/rocm_smi.cc | 241 +++++++- projects/amdsmi/src/rocm_smi_device.cc | 5 +- projects/amdsmi/src/rocm_smi_main.cc | 12 +- .../functional/evt_notif_read_write.cc | 146 +++++ .../functional/evt_notif_read_write.h | 73 +++ projects/amdsmi/tests/rocm_smi_test/main.cc | 5 + .../amdsmi/tests/rocm_smi_test/test_utils.cc | 9 +- .../amdsmi/tests/rocm_smi_test/test_utils.h | 2 + 13 files changed, 1190 insertions(+), 22 deletions(-) create mode 100755 projects/amdsmi/include/rocm_smi/kfd_ioctl.h create mode 100755 projects/amdsmi/tests/rocm_smi_test/functional/evt_notif_read_write.cc create mode 100755 projects/amdsmi/tests/rocm_smi_test/functional/evt_notif_read_write.h diff --git a/projects/amdsmi/CMakeLists.txt b/projects/amdsmi/CMakeLists.txt index 6baef76322..961515fa18 100755 --- a/projects/amdsmi/CMakeLists.txt +++ b/projects/amdsmi/CMakeLists.txt @@ -171,6 +171,8 @@ install(TARGETS ${ROCM_SMI_TARGET} LIBRARY DESTINATION ${ROCM_SMI}/lib COMPONENT ${ROCM_SMI_COMPONENT}) install(FILES ${SOURCE_DIR}/include/rocm_smi/rocm_smi.h DESTINATION rocm_smi/include/rocm_smi) +install(FILES ${SOURCE_DIR}/include/rocm_smi/kfd_ioctl.h + DESTINATION rocm_smi/include/rocm_smi) ## Add the packaging directives for the runtime library. diff --git a/projects/amdsmi/include/rocm_smi/kfd_ioctl.h b/projects/amdsmi/include/rocm_smi/kfd_ioctl.h new file mode 100755 index 0000000000..8bf56c926f --- /dev/null +++ b/projects/amdsmi/include/rocm_smi/kfd_ioctl.h @@ -0,0 +1,567 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef INCLUDE_ROCM_SMI_KFD_IOCTL_H_ +#define INCLUDE_ROCM_SMI_KFD_IOCTL_H_ + +#include +#include + +/* + * - 1.1 - initial version + * - 1.3 - Add SMI events support + */ +#define KFD_IOCTL_MAJOR_VERSION 1 +#define KFD_IOCTL_MINOR_VERSION 3 + +struct kfd_ioctl_get_version_args { + __u32 major_version; /* from KFD */ + __u32 minor_version; /* from KFD */ +}; + +/* For kfd_ioctl_create_queue_args.queue_type. */ +#define KFD_IOC_QUEUE_TYPE_COMPUTE 0x0 +#define KFD_IOC_QUEUE_TYPE_SDMA 0x1 +#define KFD_IOC_QUEUE_TYPE_COMPUTE_AQL 0x2 +#define KFD_IOC_QUEUE_TYPE_SDMA_XGMI 0x3 + +#define KFD_MAX_QUEUE_PERCENTAGE 100 +#define KFD_MAX_QUEUE_PRIORITY 15 + +struct kfd_ioctl_create_queue_args { + __u64 ring_base_address; /* to KFD */ + __u64 write_pointer_address; /* from KFD */ + __u64 read_pointer_address; /* from KFD */ + __u64 doorbell_offset; /* from KFD */ + + __u32 ring_size; /* to KFD */ + __u32 gpu_id; /* to KFD */ + __u32 queue_type; /* to KFD */ + __u32 queue_percentage; /* to KFD */ + __u32 queue_priority; /* to KFD */ + __u32 queue_id; /* from KFD */ + + __u64 eop_buffer_address; /* to KFD */ + __u64 eop_buffer_size; /* to KFD */ + __u64 ctx_save_restore_address; /* to KFD */ + __u32 ctx_save_restore_size; /* to KFD */ + __u32 ctl_stack_size; /* to KFD */ +}; + +struct kfd_ioctl_destroy_queue_args { + __u32 queue_id; /* to KFD */ + __u32 pad; +}; + +struct kfd_ioctl_update_queue_args { + __u64 ring_base_address; /* to KFD */ + + __u32 queue_id; /* to KFD */ + __u32 ring_size; /* to KFD */ + __u32 queue_percentage; /* to KFD */ + __u32 queue_priority; /* to KFD */ +}; + +struct kfd_ioctl_set_cu_mask_args { + __u32 queue_id; /* to KFD */ + __u32 num_cu_mask; /* to KFD */ + __u64 cu_mask_ptr; /* to KFD */ +}; + +struct kfd_ioctl_get_queue_wave_state_args { + __u64 ctl_stack_address; /* to KFD */ + __u32 ctl_stack_used_size; /* from KFD */ + __u32 save_area_used_size; /* from KFD */ + __u32 queue_id; /* to KFD */ + __u32 pad; +}; + +/* For kfd_ioctl_set_memory_policy_args.default_policy and alternate_policy */ +#define KFD_IOC_CACHE_POLICY_COHERENT 0 +#define KFD_IOC_CACHE_POLICY_NONCOHERENT 1 + +struct kfd_ioctl_set_memory_policy_args { + __u64 alternate_aperture_base; /* to KFD */ + __u64 alternate_aperture_size; /* to KFD */ + + __u32 gpu_id; /* to KFD */ + __u32 default_policy; /* to KFD */ + __u32 alternate_policy; /* to KFD */ + __u32 pad; +}; + +/* + * All counters are monotonic. They are used for profiling of compute jobs. + * The profiling is done by userspace. + * + * In case of GPU reset, the counter should not be affected. + */ + +struct kfd_ioctl_get_clock_counters_args { + __u64 gpu_clock_counter; /* from KFD */ + __u64 cpu_clock_counter; /* from KFD */ + __u64 system_clock_counter; /* from KFD */ + __u64 system_clock_freq; /* from KFD */ + + __u32 gpu_id; /* to KFD */ + __u32 pad; +}; + +struct kfd_process_device_apertures { + __u64 lds_base; /* from KFD */ + __u64 lds_limit; /* from KFD */ + __u64 scratch_base; /* from KFD */ + __u64 scratch_limit; /* from KFD */ + __u64 gpuvm_base; /* from KFD */ + __u64 gpuvm_limit; /* from KFD */ + __u32 gpu_id; /* from KFD */ + __u32 pad; +}; + +/* + * AMDKFD_IOC_GET_PROCESS_APERTURES is deprecated. Use + * AMDKFD_IOC_GET_PROCESS_APERTURES_NEW instead, which supports an + * unlimited number of GPUs. + */ +#define NUM_OF_SUPPORTED_GPUS 7 +struct kfd_ioctl_get_process_apertures_args { + struct kfd_process_device_apertures + process_apertures[NUM_OF_SUPPORTED_GPUS];/* from KFD */ + + /* from KFD, should be in the range [1 - NUM_OF_SUPPORTED_GPUS] */ + __u32 num_of_nodes; + __u32 pad; +}; + +struct kfd_ioctl_get_process_apertures_new_args { + /* User allocated. Pointer to struct kfd_process_device_apertures + * filled in by Kernel + */ + __u64 kfd_process_device_apertures_ptr; + /* to KFD - indicates amount of memory present in + * kfd_process_device_apertures_ptr + * from KFD - Number of entries filled by KFD. + */ + __u32 num_of_nodes; + __u32 pad; +}; + +#define MAX_ALLOWED_NUM_POINTS 100 +#define MAX_ALLOWED_AW_BUFF_SIZE 4096 +#define MAX_ALLOWED_WAC_BUFF_SIZE 128 + +struct kfd_ioctl_dbg_register_args { + __u32 gpu_id; /* to KFD */ + __u32 pad; +}; + +struct kfd_ioctl_dbg_unregister_args { + __u32 gpu_id; /* to KFD */ + __u32 pad; +}; + +struct kfd_ioctl_dbg_address_watch_args { + __u64 content_ptr; /* a pointer to the actual content */ + __u32 gpu_id; /* to KFD */ + __u32 buf_size_in_bytes; /*including gpu_id and buf_size */ +}; + +struct kfd_ioctl_dbg_wave_control_args { + __u64 content_ptr; /* a pointer to the actual content */ + __u32 gpu_id; /* to KFD */ + __u32 buf_size_in_bytes; /*including gpu_id and buf_size */ +}; + +/* Matching HSA_EVENTTYPE */ +#define KFD_IOC_EVENT_SIGNAL 0 +#define KFD_IOC_EVENT_NODECHANGE 1 +#define KFD_IOC_EVENT_DEVICESTATECHANGE 2 +#define KFD_IOC_EVENT_HW_EXCEPTION 3 +#define KFD_IOC_EVENT_SYSTEM_EVENT 4 +#define KFD_IOC_EVENT_DEBUG_EVENT 5 +#define KFD_IOC_EVENT_PROFILE_EVENT 6 +#define KFD_IOC_EVENT_QUEUE_EVENT 7 +#define KFD_IOC_EVENT_MEMORY 8 + +#define KFD_IOC_WAIT_RESULT_COMPLETE 0 +#define KFD_IOC_WAIT_RESULT_TIMEOUT 1 +#define KFD_IOC_WAIT_RESULT_FAIL 2 + +#define KFD_SIGNAL_EVENT_LIMIT 4096 + +/* For kfd_event_data.hw_exception_data.reset_type. */ +#define KFD_HW_EXCEPTION_WHOLE_GPU_RESET 0 +#define KFD_HW_EXCEPTION_PER_ENGINE_RESET 1 + +/* For kfd_event_data.hw_exception_data.reset_cause. */ +#define KFD_HW_EXCEPTION_GPU_HANG 0 +#define KFD_HW_EXCEPTION_ECC 1 + +/* For kfd_hsa_memory_exception_data.ErrorType */ +#define KFD_MEM_ERR_NO_RAS 0 +#define KFD_MEM_ERR_SRAM_ECC 1 +#define KFD_MEM_ERR_POISON_CONSUMED 2 +#define KFD_MEM_ERR_GPU_HANG 3 + +struct kfd_ioctl_create_event_args { + __u64 event_page_offset; /* from KFD */ + __u32 event_trigger_data; /* from KFD - signal events only */ + __u32 event_type; /* to KFD */ + __u32 auto_reset; /* to KFD */ + __u32 node_id; /* to KFD - only valid for certain event types */ + __u32 event_id; /* from KFD */ + __u32 event_slot_index; /* from KFD */ +}; + +struct kfd_ioctl_destroy_event_args { + __u32 event_id; /* to KFD */ + __u32 pad; +}; + +struct kfd_ioctl_set_event_args { + __u32 event_id; /* to KFD */ + __u32 pad; +}; + +struct kfd_ioctl_reset_event_args { + __u32 event_id; /* to KFD */ + __u32 pad; +}; + +struct kfd_memory_exception_failure { + __u32 NotPresent; /* Page not present or supervisor privilege */ + __u32 ReadOnly; /* Write access to a read-only page */ + __u32 NoExecute; /* Execute access to a page marked NX */ + __u32 imprecise; /* Can't determine the exact fault address */ +}; + +/* memory exception data*/ +struct kfd_hsa_memory_exception_data { + struct kfd_memory_exception_failure failure; + __u64 va; + __u32 gpu_id; + __u32 ErrorType; // 0 = no RAS error, + // 1 = ECC_SRAM, + // 2 = Link_SYNFLOOD (poison), + // 3 = GPU hang (not attributable to a specific cause), + // other values reserved +}; + +/* hw exception data */ +struct kfd_hsa_hw_exception_data { + __u32 reset_type; + __u32 reset_cause; + __u32 memory_lost; + __u32 gpu_id; +}; + +/* Event data */ +struct kfd_event_data { + union { + struct kfd_hsa_memory_exception_data memory_exception_data; + struct kfd_hsa_hw_exception_data hw_exception_data; + }; /* From KFD */ + __u64 kfd_event_data_ext; // pointer to an extension structure + // for future exception types + __u32 event_id; /* to KFD */ + __u32 pad; +}; + +struct kfd_ioctl_wait_events_args { + __u64 events_ptr; // pointed to struct + // kfd_event_data array, to KFD + __u32 num_events; /* to KFD */ + __u32 wait_for_all; /* to KFD */ + __u32 timeout; /* to KFD */ + __u32 wait_result; /* from KFD */ +}; + +struct kfd_ioctl_set_scratch_backing_va_args { + __u64 va_addr; /* to KFD */ + __u32 gpu_id; /* to KFD */ + __u32 pad; +}; + +struct kfd_ioctl_get_tile_config_args { + /* to KFD: pointer to tile array */ + __u64 tile_config_ptr; + /* to KFD: pointer to macro tile array */ + __u64 macro_tile_config_ptr; + /* to KFD: array size allocated by user mode + * from KFD: array size filled by kernel + */ + __u32 num_tile_configs; + /* to KFD: array size allocated by user mode + * from KFD: array size filled by kernel + */ + __u32 num_macro_tile_configs; + + __u32 gpu_id; /* to KFD */ + __u32 gb_addr_config; /* from KFD */ + __u32 num_banks; /* from KFD */ + __u32 num_ranks; /* from KFD */ + /* struct size can be extended later if needed + * without breaking ABI compatibility + */ +}; + +struct kfd_ioctl_set_trap_handler_args { + __u64 tba_addr; /* to KFD */ + __u64 tma_addr; /* to KFD */ + __u32 gpu_id; /* to KFD */ + __u32 pad; +}; + +struct kfd_ioctl_acquire_vm_args { + __u32 drm_fd; /* to KFD */ + __u32 gpu_id; /* to KFD */ +}; + +/* Allocation flags: memory types */ +#define KFD_IOC_ALLOC_MEM_FLAGS_VRAM (1 << 0) +#define KFD_IOC_ALLOC_MEM_FLAGS_GTT (1 << 1) +#define KFD_IOC_ALLOC_MEM_FLAGS_USERPTR (1 << 2) +#define KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL (1 << 3) +#define KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP (1 << 4) +/* Allocation flags: attributes/access options */ +#define KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE (1 << 31) +#define KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE (1 << 30) +#define KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC (1 << 29) +#define KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE (1 << 28) +#define KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM (1 << 27) +#define KFD_IOC_ALLOC_MEM_FLAGS_COHERENT (1 << 26) + +/* Allocate memory for later SVM (shared virtual memory) mapping. + * + * @va_addr: virtual address of the memory to be allocated + * all later mappings on all GPUs will use this address + * @size: size in bytes + * @handle: buffer handle returned to user mode, used to refer to + * this allocation for mapping, unmapping and freeing + * @mmap_offset: for CPU-mapping the allocation by mmapping a render node + * for userptrs this is overloaded to specify the CPU address + * @gpu_id: device identifier + * @flags: memory type and attributes. See KFD_IOC_ALLOC_MEM_FLAGS above + */ +struct kfd_ioctl_alloc_memory_of_gpu_args { + __u64 va_addr; /* to KFD */ + __u64 size; /* to KFD */ + __u64 handle; /* from KFD */ + __u64 mmap_offset; /* to KFD (userptr), from KFD (mmap offset) */ + __u32 gpu_id; /* to KFD */ + __u32 flags; +}; + +/* Free memory allocated with kfd_ioctl_alloc_memory_of_gpu + * + * @handle: memory handle returned by alloc + */ +struct kfd_ioctl_free_memory_of_gpu_args { + __u64 handle; /* to KFD */ +}; + +/* Map memory to one or more GPUs + * + * @handle: memory handle returned by alloc + * @device_ids_array_ptr: array of gpu_ids (__u32 per device) + * @n_devices: number of devices in the array + * @n_success: number of devices mapped successfully + * + * @n_success returns information to the caller how many devices from + * the start of the array have mapped the buffer successfully. It can + * be passed into a subsequent retry call to skip those devices. For + * the first call the caller should initialize it to 0. + * + * If the ioctl completes with return code 0 (success), n_success == + * n_devices. + */ +struct kfd_ioctl_map_memory_to_gpu_args { + __u64 handle; /* to KFD */ + __u64 device_ids_array_ptr; /* to KFD */ + __u32 n_devices; /* to KFD */ + __u32 n_success; /* to/from KFD */ +}; + +/* Unmap memory from one or more GPUs + * + * same arguments as for mapping + */ +struct kfd_ioctl_unmap_memory_from_gpu_args { + __u64 handle; /* to KFD */ + __u64 device_ids_array_ptr; /* to KFD */ + __u32 n_devices; /* to KFD */ + __u32 n_success; /* to/from KFD */ +}; + +/* Allocate GWS for specific queue + * + * @queue_id: queue's id that GWS is allocated for + * @num_gws: how many GWS to allocate + * @first_gws: index of the first GWS allocated. + * only support contiguous GWS allocation + */ +struct kfd_ioctl_alloc_queue_gws_args { + __u32 queue_id; /* to KFD */ + __u32 num_gws; /* to KFD */ + __u32 first_gws; /* from KFD */ + __u32 pad; +}; + +struct kfd_ioctl_get_dmabuf_info_args { + __u64 size; /* from KFD */ + __u64 metadata_ptr; /* to KFD */ + __u32 metadata_size; // to KFD (space allocated by user) + // from KFD (actual metadata size) + __u32 gpu_id; /* from KFD */ + __u32 flags; /* from KFD (KFD_IOC_ALLOC_MEM_FLAGS) */ + __u32 dmabuf_fd; /* to KFD */ +}; + +struct kfd_ioctl_import_dmabuf_args { + __u64 va_addr; /* to KFD */ + __u64 handle; /* from KFD */ + __u32 gpu_id; /* to KFD */ + __u32 dmabuf_fd; /* to KFD */ +}; + +/* + * KFD SMI(System Management Interface) events + */ +/* Event type (defined by bitmask) */ +#define KFD_SMI_EVENT_VMFAULT 0x0000000000000001 + +struct kfd_ioctl_smi_events_args { + __u32 gpuid; /* to KFD */ + __u32 anon_fd; /* from KFD */ +}; + +/* Register offset inside the remapped mmio page + */ +enum kfd_mmio_remap { + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL = 0, + KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL = 4, +}; + +#define AMDKFD_IOCTL_BASE 'K' +#define AMDKFD_IO(nr) _IO(AMDKFD_IOCTL_BASE, nr) +#define AMDKFD_IOR(nr, type) _IOR(AMDKFD_IOCTL_BASE, nr, type) +#define AMDKFD_IOW(nr, type) _IOW(AMDKFD_IOCTL_BASE, nr, type) +#define AMDKFD_IOWR(nr, type) _IOWR(AMDKFD_IOCTL_BASE, nr, type) + +#define AMDKFD_IOC_GET_VERSION \ + AMDKFD_IOR(0x01, struct kfd_ioctl_get_version_args) + +#define AMDKFD_IOC_CREATE_QUEUE \ + AMDKFD_IOWR(0x02, struct kfd_ioctl_create_queue_args) + +#define AMDKFD_IOC_DESTROY_QUEUE \ + AMDKFD_IOWR(0x03, struct kfd_ioctl_destroy_queue_args) + +#define AMDKFD_IOC_SET_MEMORY_POLICY \ + AMDKFD_IOW(0x04, struct kfd_ioctl_set_memory_policy_args) + +#define AMDKFD_IOC_GET_CLOCK_COUNTERS \ + AMDKFD_IOWR(0x05, struct kfd_ioctl_get_clock_counters_args) + +#define AMDKFD_IOC_GET_PROCESS_APERTURES \ + AMDKFD_IOR(0x06, struct kfd_ioctl_get_process_apertures_args) + +#define AMDKFD_IOC_UPDATE_QUEUE \ + AMDKFD_IOW(0x07, struct kfd_ioctl_update_queue_args) + +#define AMDKFD_IOC_CREATE_EVENT \ + AMDKFD_IOWR(0x08, struct kfd_ioctl_create_event_args) + +#define AMDKFD_IOC_DESTROY_EVENT \ + AMDKFD_IOW(0x09, struct kfd_ioctl_destroy_event_args) + +#define AMDKFD_IOC_SET_EVENT \ + AMDKFD_IOW(0x0A, struct kfd_ioctl_set_event_args) + +#define AMDKFD_IOC_RESET_EVENT \ + AMDKFD_IOW(0x0B, struct kfd_ioctl_reset_event_args) + +#define AMDKFD_IOC_WAIT_EVENTS \ + AMDKFD_IOWR(0x0C, struct kfd_ioctl_wait_events_args) + +#define AMDKFD_IOC_DBG_REGISTER \ + AMDKFD_IOW(0x0D, struct kfd_ioctl_dbg_register_args) + +#define AMDKFD_IOC_DBG_UNREGISTER \ + AMDKFD_IOW(0x0E, struct kfd_ioctl_dbg_unregister_args) + +#define AMDKFD_IOC_DBG_ADDRESS_WATCH \ + AMDKFD_IOW(0x0F, struct kfd_ioctl_dbg_address_watch_args) + +#define AMDKFD_IOC_DBG_WAVE_CONTROL \ + AMDKFD_IOW(0x10, struct kfd_ioctl_dbg_wave_control_args) + +#define AMDKFD_IOC_SET_SCRATCH_BACKING_VA \ + AMDKFD_IOWR(0x11, struct kfd_ioctl_set_scratch_backing_va_args) + +#define AMDKFD_IOC_GET_TILE_CONFIG \ + AMDKFD_IOWR(0x12, struct kfd_ioctl_get_tile_config_args) + +#define AMDKFD_IOC_SET_TRAP_HANDLER \ + AMDKFD_IOW(0x13, struct kfd_ioctl_set_trap_handler_args) + +#define AMDKFD_IOC_GET_PROCESS_APERTURES_NEW \ + AMDKFD_IOWR(0x14, \ + struct kfd_ioctl_get_process_apertures_new_args) + +#define AMDKFD_IOC_ACQUIRE_VM \ + AMDKFD_IOW(0x15, struct kfd_ioctl_acquire_vm_args) + +#define AMDKFD_IOC_ALLOC_MEMORY_OF_GPU \ + AMDKFD_IOWR(0x16, struct kfd_ioctl_alloc_memory_of_gpu_args) + +#define AMDKFD_IOC_FREE_MEMORY_OF_GPU \ + AMDKFD_IOW(0x17, struct kfd_ioctl_free_memory_of_gpu_args) + +#define AMDKFD_IOC_MAP_MEMORY_TO_GPU \ + AMDKFD_IOWR(0x18, struct kfd_ioctl_map_memory_to_gpu_args) + +#define AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU \ + AMDKFD_IOWR(0x19, struct kfd_ioctl_unmap_memory_from_gpu_args) + +#define AMDKFD_IOC_SET_CU_MASK \ + AMDKFD_IOW(0x1A, struct kfd_ioctl_set_cu_mask_args) + +#define AMDKFD_IOC_GET_QUEUE_WAVE_STATE \ + AMDKFD_IOWR(0x1B, struct kfd_ioctl_get_queue_wave_state_args) + +#define AMDKFD_IOC_GET_DMABUF_INFO \ + AMDKFD_IOWR(0x1C, struct kfd_ioctl_get_dmabuf_info_args) + +#define AMDKFD_IOC_IMPORT_DMABUF \ + AMDKFD_IOWR(0x1D, struct kfd_ioctl_import_dmabuf_args) + +#define AMDKFD_IOC_ALLOC_QUEUE_GWS \ + AMDKFD_IOWR(0x1E, struct kfd_ioctl_alloc_queue_gws_args) + +#define AMDKFD_IOC_SMI_EVENTS \ + AMDKFD_IOWR(0x1F, struct kfd_ioctl_smi_events_args) + +#define AMDKFD_COMMAND_START 0x01 +#define AMDKFD_COMMAND_END 0x20 + +#endif // INCLUDE_ROCM_SMI_KFD_IOCTL_H_ diff --git a/projects/amdsmi/include/rocm_smi/rocm_smi.h b/projects/amdsmi/include/rocm_smi/rocm_smi.h index 5bc4f0e5c3..d510ff3e96 100755 --- a/projects/amdsmi/include/rocm_smi/rocm_smi.h +++ b/projects/amdsmi/include/rocm_smi/rocm_smi.h @@ -52,6 +52,12 @@ extern "C" { #include +// In a file included from kfd_ioctl.h, is a variable called +// c++ keyword "virtual" +#define virtual virtual_tmp +#include "rocm_smi/kfd_ioctl.h" +#undef virtual + /** \file rocm_smi.h * Main header file for the ROCm SMI library. * All required function, structure, enum, etc. definitions should be defined @@ -252,6 +258,29 @@ typedef struct { uint64_t time_running; //!< Time that che counter was running } rsmi_counter_value_t; +/* + * Event notification event types + * See + */ +typedef enum { + RSMI_EVT_NOTIF_VMFAULT = KFD_SMI_EVENT_VMFAULT, //!< VM page fault + RSMI_EVT_NOTIF_FIRST = RSMI_EVT_NOTIF_VMFAULT, + + RSMI_EVT_NOTIF_LAST = RSMI_EVT_NOTIF_VMFAULT +} rsmi_evt_notification_type_t; + +//! Maximum number of characters an event notification message will be +#define MAX_EVENT_NOTIFICATION_MSG_SIZE 64 + +/** + * Event notification data returned from event notification API + */ +typedef struct { + uint32_t dv_ind; //!< Index of device that corresponds to the event + rsmi_evt_notification_type_t event; //!< Event type + char message[MAX_EVENT_NOTIFICATION_MSG_SIZE]; // Event message +} rsmi_evt_notification_data_t; + /** * Clock types */ @@ -2813,6 +2842,108 @@ rsmi_func_iter_value_get(rsmi_func_id_iter_handle_t handle, /** @} */ // end of APISupport +/** + * @brief Prepare to collect event notifications for a GPU + * + * @param dv_ind a device index corresponding to the device on which to + * listen for events + * + * @details This function prepares to collect events for the GPU with device + * ID @p dv_ind, by initializing any required system parameters. This call + * may open files which will remain open until ::rsmi_event_notification_stop() + * is called. + * + * @param[in] unused The parameter is currently ignored + * + * @retval ::RSMI_STATUS_SUCCESS is returned upon successful call. + */ +rsmi_status_t +rsmi_event_notification_init(uint32_t dv_ind); + +/** + * @brief Specify which events to collect for a device + * + * @details Given a device index @p dv_ind and a @p mask consisting of + * elements of ::rsmi_evt_notification_type_t OR'd together, this function + * will listen for the events specified in @p mask on the device + * corresponding to @p dv_ind. + * + * @param dv_ind a device index corresponding to the device on which to + * listen for events + * + * @param mask 0 or more elements of ::rsmi_evt_notification_type_t OR'd + * together that indicate which event types to listen for. + * + * @retval ::RSMI_INITIALIZATION_ERROR is returned if + * ::rsmi_event_notification_init() has not been called before a call to this + * function + * + * @retval ::RSMI_STATUS_SUCCESS is returned upon successful call + */ +rsmi_status_t +rsmi_event_notification_mask_set(uint32_t dv_ind, uint64_t mask); + +/** + * @brief Collect event notifications, waiting a specified amount of time + * + * @details Given a time period @p timeout_ms in milliseconds and a caller- + * provided buffer of ::rsmi_evt_notification_data_t's @p data with a length + * (in ::rsmi_evt_notification_data_t's, also specified by the caller) in the + * memory location pointed to by @p num_elem, this function will collect + * ::rsmi_evt_notification_type_t events for up to @p timeout_ms milliseconds, + * and write up to *@p num_elem event items to @p data. Upon return @p num_elem + * is updated with the number of events that were actually written. If events + * are already present when this function is called, it will write the events + * to the buffer then poll for new events if there is still caller-provided + * buffer available to write any new events that would be found. + * + * This function requires prior calls to ::rsmi_event_notification_init() and + * ::rsmi_event_notification_mask_set(). This function polls for the + * occurrance of the events on the respective devices that were previously + * specified by ::rsmi_event_notification_mask_set(). + * + * @param[inout] num_elem pointer to uint32_t, provided by the caller. On + * input, this value tells how many ::rsmi_evt_notification_data_t elements + * are being provided by the caller with @p data. On output, the location + * pointed to by @p num_elem will contain the number of items written to + * the provided buffer. + * + * @param[out] data pointer to a caller-provided memory buffer of size + * @p num_elem ::rsmi_evt_notification_data_t to which this function may safely + * write. If there are events found, up to @p num_elem event items will be + * written to @p data. + * + * @retval ::RSMI_STATUS_SUCCESS The function ran successfully. The events + * that were found are written to @p data and @p num_elems is updated + * with the number of elements that were written. + * + * @retval ::RSMI_STATUS_NO_DATA No events were found to collect. + * + */ +rsmi_status_t +rsmi_event_notification_get(int timout_ms, + uint32_t *num_elem, rsmi_evt_notification_data_t *data); + +/* + * @brief Close any file handles and free any resources used by event + * notification for a GPU + * + * @details Any resources used by event notification for the GPU with + * device index @p dv_ind will be free with this + * function. This includes freeing any memory and closing file handles. This + * should be called for every call to ::rsmi_event_notification_init() + * + * @param[in] dv_ind The device index of the GPU for which event + * notification resources will be free + * + * @retval ::RSMI_STATUS_INVALID_ARGS resources for the given device have + * either already been freed, or were never allocated by + * ::rsmi_event_notification_init() + * + * @retval ::RSMI_STATUS_SUCCESS is returned upon successful call + */ +rsmi_status_t rsmi_event_notification_stop(uint32_t dv_ind); + #ifdef __cplusplus } #endif // __cplusplus diff --git a/projects/amdsmi/include/rocm_smi/rocm_smi_device.h b/projects/amdsmi/include/rocm_smi/rocm_smi_device.h index 83395ea668..2bc6738951 100755 --- a/projects/amdsmi/include/rocm_smi/rocm_smi_device.h +++ b/projects/amdsmi/include/rocm_smi/rocm_smi_device.h @@ -191,6 +191,11 @@ class Device { SupportedFuncMap *supported_funcs(void) {return &supported_funcs_;} uint64_t kfd_gpu_id(void) const {return kfd_gpu_id_;} void set_kfd_gpu_id(uint64_t id) {kfd_gpu_id_ = id;} + + void set_evt_notif_anon_file_ptr(FILE *f) {evt_notif_anon_file_ptr_ = f;} + FILE *evt_notif_anon_file_ptr(void) const {return evt_notif_anon_file_ptr_;} + void set_evt_notif_anon_fd(int fd) {evt_notif_anon_fd_ = fd;} + int evt_notif_anon_fd(void) const {return evt_notif_anon_fd_;} void fillSupportedFuncs(void); void DumpSupportedFunctions(void); bool DeviceAPISupported(std::string name, uint64_t variant, @@ -216,6 +221,9 @@ class Device { evt::RSMIEventGrpHashFunction> supported_event_groups_; // std::map kfdNodePropMap_; SupportedFuncMap supported_funcs_; + + int evt_notif_anon_fd_; + FILE *evt_notif_anon_file_ptr_; }; } // namespace smi diff --git a/projects/amdsmi/include/rocm_smi/rocm_smi_main.h b/projects/amdsmi/include/rocm_smi/rocm_smi_main.h index e7bca3db88..bb8b326daf 100755 --- a/projects/amdsmi/include/rocm_smi/rocm_smi_main.h +++ b/projects/amdsmi/include/rocm_smi/rocm_smi_main.h @@ -53,6 +53,7 @@ #include #include #include +#include // NOLINT #include "rocm_smi/rocm_smi_kfd.h" #include "rocm_smi/rocm_smi_device.h" @@ -90,6 +91,12 @@ class RocmSMI { std::map> & kfd_node_map(void) { return kfd_node_map_;} + int kfd_notif_evt_fh(void) const {return kfd_notif_evt_fh_;} + void set_kfd_notif_evt_fh(int fd) {kfd_notif_evt_fh_ = fd;} + std::mutex *kfd_notif_evt_fh_mutex(void) {return &kfd_notif_evt_fh_mutex_;} + int kfd_notif_evt_fh_refcnt_inc() {return ++kfd_notif_evt_fh_refcnt_;} + int kfd_notif_evt_fh_refcnt_dec() {return --kfd_notif_evt_fh_refcnt_;} + private: std::vector> devices_; std::map> kfd_node_map_; @@ -105,6 +112,10 @@ class RocmSMI { RocmSMI_env_vars env_vars_; uint64_t init_options_; uint32_t euid_; + + int kfd_notif_evt_fh_; + int kfd_notif_evt_fh_refcnt_; + std::mutex kfd_notif_evt_fh_mutex_; }; } // namespace smi diff --git a/projects/amdsmi/src/rocm_smi.cc b/projects/amdsmi/src/rocm_smi.cc index 5eca70e3c4..1cb73a6f92 100755 --- a/projects/amdsmi/src/rocm_smi.cc +++ b/projects/amdsmi/src/rocm_smi.cc @@ -47,6 +47,9 @@ #include #include #include +#include +#include +#include #include #include @@ -72,6 +75,23 @@ static const uint32_t kMaxOverdriveLevel = 20; +static rsmi_status_t errno_to_rsmi_status(uint32_t err) { + switch (err) { + case 0: return RSMI_STATUS_SUCCESS; + case ESRCH: return RSMI_STATUS_NOT_FOUND; + case EACCES: return RSMI_STATUS_PERMISSION; + case EPERM: + case ENOENT: return RSMI_STATUS_NOT_SUPPORTED; + case EBADF: + case EISDIR: return RSMI_STATUS_FILE_ERROR; + case EINTR: return RSMI_STATUS_INTERRUPT; + case EIO: return RSMI_STATUS_UNEXPECTED_SIZE; + case ENXIO: return RSMI_STATUS_UNEXPECTED_DATA; + case EBUSY: return RSMI_STATUS_BUSY; + default: return RSMI_STATUS_UNKNOWN_ERROR; + } +} + static rsmi_status_t handleException() { try { throw; @@ -86,6 +106,8 @@ static rsmi_status_t handleException() { } catch (const std::nested_exception& e) { debug_print("Callback threw.\n"); return RSMI_STATUS_INTERNAL_EXCEPTION; + } catch (int erno) { + return errno_to_rsmi_status(erno); } catch (...) { debug_print("Unknown exception caught.\n"); return RSMI_STATUS_INTERNAL_EXCEPTION; @@ -170,22 +192,6 @@ static pthread_mutex_t *get_mutex(uint32_t dv_ind) { return dev->mutex(); } -static rsmi_status_t errno_to_rsmi_status(uint32_t err) { - switch (err) { - case 0: return RSMI_STATUS_SUCCESS; - case ESRCH: return RSMI_STATUS_NOT_FOUND; - case EACCES: return RSMI_STATUS_PERMISSION; - case EPERM: - case ENOENT: return RSMI_STATUS_NOT_SUPPORTED; - case EBADF: - case EISDIR: return RSMI_STATUS_FILE_ERROR; - case EINTR: return RSMI_STATUS_INTERRUPT; - case EIO: return RSMI_STATUS_UNEXPECTED_SIZE; - case ENXIO: return RSMI_STATUS_UNEXPECTED_DATA; - default: return RSMI_STATUS_UNKNOWN_ERROR; - } -} - static uint64_t get_multiplier_from_str(char units_char) { uint32_t multiplier = 0; @@ -3157,6 +3163,209 @@ rsmi_func_iter_next(rsmi_func_id_iter_handle_t handle) { CATCH } + +static bool check_evt_notif_support(int kfd_fd) { + struct kfd_ioctl_get_version_args args = {0, 0}; + + if (ioctl(kfd_fd, AMDKFD_IOC_GET_VERSION, &args) == -1) { + return RSMI_STATUS_INIT_ERROR; + } + + if (args.major_version < 2 && args.minor_version < 3) { + return false; + } + return true; +} + +static const char *kPathKFDIoctl = "/dev/kfd"; + +rsmi_status_t +rsmi_event_notification_init(uint32_t dv_ind) { + TRY + GET_DEV_FROM_INDX + + std::lock_guard guard(*smi.kfd_notif_evt_fh_mutex()); + if (smi.kfd_notif_evt_fh() == -1) { + int kfd_fd = open(kPathKFDIoctl, O_RDWR | O_CLOEXEC); + + if (kfd_fd <= 0) { + return RSMI_STATUS_FILE_ERROR; + } + + if (!check_evt_notif_support(kfd_fd)) { + close(kfd_fd); + return RSMI_STATUS_NOT_SUPPORTED; + } + + smi.set_kfd_notif_evt_fh(kfd_fd); + } + smi.kfd_notif_evt_fh_refcnt_inc(); + + struct kfd_ioctl_smi_events_args args; + + assert(dev->kfd_gpu_id() <= UINT32_MAX); + args.gpuid = static_cast(dev->kfd_gpu_id()); + + int ret = ioctl(smi.kfd_notif_evt_fh(), AMDKFD_IOC_SMI_EVENTS, &args); + if (ret < 0) { + return errno_to_rsmi_status(errno); + } + if (args.anon_fd < 1) { + return RSMI_STATUS_NO_DATA; + } + + dev->set_evt_notif_anon_fd(args.anon_fd); + FILE *anon_file_ptr = fdopen(args.anon_fd, "r"); + if (anon_file_ptr == nullptr) { + close(dev->evt_notif_anon_fd()); + return errno_to_rsmi_status(errno); + } + dev->set_evt_notif_anon_file_ptr(anon_file_ptr); + + return RSMI_STATUS_SUCCESS; + + CATCH +} + +rsmi_status_t +rsmi_event_notification_mask_set(uint32_t dv_ind, uint64_t mask) { + TRY + GET_DEV_FROM_INDX + + if (dev->evt_notif_anon_fd() == -1) { + return RSMI_INITIALIZATION_ERROR; + } + ssize_t ret = write(dev->evt_notif_anon_fd(), &mask, sizeof(uint64_t)); + + if (ret == -1) { + return errno_to_rsmi_status(errno); + } + + return RSMI_STATUS_SUCCESS; + CATCH +} + +rsmi_status_t +rsmi_event_notification_get(int timeout_ms, + uint32_t *num_elem, rsmi_evt_notification_data_t *data) { + TRY + + if (num_elem == nullptr || data == nullptr || *num_elem == 0) { + return RSMI_STATUS_INVALID_ARGS; + } + + uint32_t buffer_size = *num_elem; + *num_elem = 0; + + rsmi_evt_notification_data_t *data_item; + + // struct pollfd { + // int fd; /* file descriptor */ + // short events; /* requested events */ + // short revents; /* returned events */ + // }; + std::vector fds; + amd::smi::RocmSMI& smi = amd::smi::RocmSMI::getInstance(); + std::vector fd_indx_to_dev_id; + + for (uint32_t i = 0; i < smi.monitor_devices().size(); ++i) { + if (smi.monitor_devices()[i]->evt_notif_anon_fd() == -1) { + continue; + } + fds.push_back({smi.monitor_devices()[i]->evt_notif_anon_fd(), + POLLIN | POLLRDNORM, 0}); + fd_indx_to_dev_id.push_back(i); + } + + auto fill_data_buffer = [&](bool did_poll) { + for (uint32_t i = 0; i < fds.size(); ++i) { + if (did_poll) { + if (!(fds[i].revents & (POLLIN | POLLRDNORM))) { + continue; + } + } + + if (*num_elem >= buffer_size) { + return; + } + + FILE *anon_fp = + smi.monitor_devices()[fd_indx_to_dev_id[i]]->evt_notif_anon_file_ptr(); + data_item = + reinterpret_cast(&data[*num_elem]); + + uint64_t event; + while (fscanf(anon_fp, "%lx %63s\n", &event, + reinterpret_cast(&data_item->message)) == 2) { + /* Output is in format as "event information\n" + * Both event are expressed in hex. + * information is a string + */ + data_item->event = (rsmi_evt_notification_type_t)event; + data_item->dv_ind = fd_indx_to_dev_id[i]; + ++(*num_elem); + + if (*num_elem >= buffer_size) { + break; + } + data_item = + reinterpret_cast(&data[*num_elem]); + } + } + }; + + // Collect any left-over events from a poll in a previous call to + // rsmi_event_notification_get() + fill_data_buffer(false); + + if (*num_elem < buffer_size && errno != EAGAIN) { + return errno_to_rsmi_status(errno); + } else if (*num_elem >= buffer_size) { + return RSMI_STATUS_SUCCESS; + } + + // We still have buffer left, see if there are any new events + int p_ret = poll(fds.data(), fds.size(), timeout_ms); + if (p_ret > 0) { + fill_data_buffer(true); + } else if (p_ret < 0) { + return errno_to_rsmi_status(errno); + } + if (*num_elem == 0) { + return RSMI_STATUS_NO_DATA; + } + + return RSMI_STATUS_SUCCESS; + CATCH +} + +rsmi_status_t rsmi_event_notification_stop(uint32_t dv_ind) { + TRY + GET_DEV_FROM_INDX + std::lock_guard guard(*smi.kfd_notif_evt_fh_mutex()); + + if (dev->evt_notif_anon_fd() == -1) { + return RSMI_STATUS_INVALID_ARGS; + } +// close(dev->evt_notif_anon_fd()); + FILE *anon_fp = smi.monitor_devices()[dv_ind]->evt_notif_anon_file_ptr(); + fclose(anon_fp); + assert(errno == 0 || errno == EAGAIN); + dev->set_evt_notif_anon_file_ptr(nullptr); + dev->set_evt_notif_anon_fd(-1); + + if (!smi.kfd_notif_evt_fh_refcnt_dec()) { + int ret = close(smi.kfd_notif_evt_fh()); + smi.set_kfd_notif_evt_fh(-1); + if (ret < 0) { + return errno_to_rsmi_status(errno); + } + } + + return RSMI_STATUS_SUCCESS; + CATCH +} + // UNDOCUMENTED FUNCTIONS // This functions are not declared in rocm_smi.h. They are either not fully // supported, or to be used for test purposes. diff --git a/projects/amdsmi/src/rocm_smi_device.cc b/projects/amdsmi/src/rocm_smi_device.cc index 131fd273c4..7760b7330f 100755 --- a/projects/amdsmi/src/rocm_smi_device.cc +++ b/projects/amdsmi/src/rocm_smi_device.cc @@ -458,9 +458,8 @@ static const std::map kDevFuncDependsMap = { if (X) return X; \ } -Device::Device(std::string p, RocmSMI_env_vars const *e) : path_(p), env_(e) { - monitor_ = nullptr; - +Device::Device(std::string p, RocmSMI_env_vars const *e) : monitor_(nullptr), + path_(p), env_(e), evt_notif_anon_fd_(-1) { #ifdef NDEBUG env_ = nullptr; #endif diff --git a/projects/amdsmi/src/rocm_smi_main.cc b/projects/amdsmi/src/rocm_smi_main.cc index 4846d8decb..9cd0224439 100755 --- a/projects/amdsmi/src/rocm_smi_main.cc +++ b/projects/amdsmi/src/rocm_smi_main.cc @@ -266,7 +266,7 @@ RocmSMI::Initialize(uint64_t flags) { if (ret != 0) { throw amd::smi::rsmi_exception(RSMI_INITIALIZATION_ERROR, - "Failed to initialize rocm_smi library (amdgpu node discovery."); + "Failed to initialize rocm_smi library (amdgpu node discovery)."); } std::map> tmp_map; @@ -299,12 +299,20 @@ RocmSMI::Initialize(uint64_t flags) { void RocmSMI::Cleanup() { + if (kfd_notif_evt_fh() >= 0) { + int ret = close(kfd_notif_evt_fh()); + if (ret < 0) { + throw amd::smi::rsmi_exception(RSMI_STATUS_FILE_ERROR, + "Failed to close kfd file handle on shutdown."); + } + } s_monitor_devices.clear(); devices_.clear(); monitors_.clear(); } -RocmSMI::RocmSMI(uint64_t flags) : init_options_(flags) { +RocmSMI::RocmSMI(uint64_t flags) : init_options_(flags), + kfd_notif_evt_fh_(-1), kfd_notif_evt_fh_refcnt_(0) { } RocmSMI::~RocmSMI() { diff --git a/projects/amdsmi/tests/rocm_smi_test/functional/evt_notif_read_write.cc b/projects/amdsmi/tests/rocm_smi_test/functional/evt_notif_read_write.cc new file mode 100755 index 0000000000..19d6d10538 --- /dev/null +++ b/projects/amdsmi/tests/rocm_smi_test/functional/evt_notif_read_write.cc @@ -0,0 +1,146 @@ +/* + * ============================================================================= + * ROC Runtime Conformance Release License + * ============================================================================= + * The University of Illinois/NCSA + * Open Source License (NCSA) + * + * Copyright (c) 2020, Advanced Micro Devices, Inc. + * All rights reserved. + * + * Developed by: + * + * AMD Research and AMD ROC Software Development + * + * Advanced Micro Devices, Inc. + * + * www.amd.com + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal with the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * - Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimers. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimers in + * the documentation and/or other materials provided with the distribution. + * - Neither the names of , + * nor the names of its contributors may be used to endorse or promote + * products derived from this Software without specific prior written + * permission. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS WITH THE SOFTWARE. + * + */ + +#include +#include + +#include + +#include "gtest/gtest.h" +#include "rocm_smi/rocm_smi.h" +#include "rocm_smi_test/functional/evt_notif_read_write.h" +#include "rocm_smi_test/test_common.h" +#include "rocm_smi_test/test_utils.h" + +TestEvtNotifReadWrite::TestEvtNotifReadWrite() : TestBase() { + set_title("RSMI Event Notification Read/Write Test"); + set_description("The Event Notification Read/Write tests verifies that " + "we can configure to collect various event types and then read them"); +} + +TestEvtNotifReadWrite::~TestEvtNotifReadWrite(void) { +} + +void TestEvtNotifReadWrite::SetUp(void) { + TestBase::SetUp(); + return; +} + +void TestEvtNotifReadWrite::DisplayTestInfo(void) { + TestBase::DisplayTestInfo(); +} + +void TestEvtNotifReadWrite::DisplayResults(void) const { + TestBase::DisplayResults(); + return; +} + +void TestEvtNotifReadWrite::Close() { + // This will close handles opened within rsmitst utility calls and call + // rsmi_shut_down(), so it should be done after other hsa cleanup + TestBase::Close(); +} + +void TestEvtNotifReadWrite::Run(void) { + rsmi_status_t ret; + uint32_t dv_ind; + + TestBase::Run(); + if (setup_failed_) { + std::cout << "** SetUp Failed for this test. Skipping.**" << std::endl; + return; + } + + rsmi_evt_notification_type_t evt_type = RSMI_EVT_NOTIF_FIRST; + uint64_t mask = evt_type; + while (evt_type != RSMI_EVT_NOTIF_LAST) { + mask |= evt_type; + evt_type = static_cast( + static_cast(evt_type)*2); + } + + for (dv_ind = 0; dv_ind < num_monitor_devs(); ++dv_ind) { + ret = rsmi_event_notification_init(dv_ind); + if (ret == RSMI_STATUS_NOT_SUPPORTED) { + std::cout << + "Event notification is not supported for this driver version." << + std::endl; + return; + } + ASSERT_EQ(ret, RSMI_STATUS_SUCCESS); + ret = rsmi_event_notification_mask_set(dv_ind, mask); + ASSERT_EQ(ret, RSMI_STATUS_SUCCESS); + } + + rsmi_evt_notification_data_t data[10]; + uint32_t num_elem = 10; + + ret = rsmi_event_notification_get(10000, &num_elem, data); + if (ret == RSMI_STATUS_SUCCESS || ret == RSMI_STATUS_INSUFFICIENT_SIZE) { + EXPECT_LE(num_elem, 10) << + "Expected the number of elements found to be <= buffer size (10)"; + for (uint32_t i = 0; i < num_elem; ++i) { + std::cout << "\tdv_ind=" << data[i].dv_ind << + " Type: " << NameFromEvtNotifType(data[i].event) << + " Mesg: " << data[i].message << std::endl; + } + if (ret == RSMI_STATUS_INSUFFICIENT_SIZE) { + std::cout << + "\t\tBuffer size is 10, but more than 10 events are available." << + std::endl; + } + } else if (ret == RSMI_STATUS_NO_DATA) { + std::cout << "\tNo events were collected." << std::endl; + } else { + // This should always fail. We want to print out the return code. + EXPECT_EQ(ret, RSMI_STATUS_SUCCESS) << + "Unexpected return code for rsmi_event_notification_get()"; + } + + for (uint32_t dv_ind = 0; dv_ind < num_monitor_devs(); ++dv_ind) { + ret = rsmi_event_notification_stop(dv_ind); + ASSERT_EQ(ret, RSMI_STATUS_SUCCESS); + } +} diff --git a/projects/amdsmi/tests/rocm_smi_test/functional/evt_notif_read_write.h b/projects/amdsmi/tests/rocm_smi_test/functional/evt_notif_read_write.h new file mode 100755 index 0000000000..3e3e142271 --- /dev/null +++ b/projects/amdsmi/tests/rocm_smi_test/functional/evt_notif_read_write.h @@ -0,0 +1,73 @@ +/* + * ============================================================================= + * ROC Runtime Conformance Release License + * ============================================================================= + * The University of Illinois/NCSA + * Open Source License (NCSA) + * + * Copyright (c) 2020, Advanced Micro Devices, Inc. + * All rights reserved. + * + * Developed by: + * + * AMD Research and AMD ROC Software Development + * + * Advanced Micro Devices, Inc. + * + * www.amd.com + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal with the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * - Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimers. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimers in + * the documentation and/or other materials provided with the distribution. + * - Neither the names of , + * nor the names of its contributors may be used to endorse or promote + * products derived from this Software without specific prior written + * permission. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS WITH THE SOFTWARE. + * + */ +#ifndef TESTS_ROCM_SMI_TEST_FUNCTIONAL_EVT_NOTIF_READ_WRITE_H_ +#define TESTS_ROCM_SMI_TEST_FUNCTIONAL_EVT_NOTIF_READ_WRITE_H_ + +#include "rocm_smi_test/test_base.h" + +class TestEvtNotifReadWrite : public TestBase { + public: + TestEvtNotifReadWrite(); + + // @Brief: Destructor for test case of TestEvtNotifReadWrite + virtual ~TestEvtNotifReadWrite(); + + // @Brief: Setup the environment for measurement + virtual void SetUp(); + + // @Brief: Core measurement execution + virtual void Run(); + + // @Brief: Clean up and retrive the resource + virtual void Close(); + + // @Brief: Display results + virtual void DisplayResults() const; + + // @Brief: Display information about what this test does + virtual void DisplayTestInfo(void); +}; + +#endif // TESTS_ROCM_SMI_TEST_FUNCTIONAL_EVT_NOTIF_READ_WRITE_H_ diff --git a/projects/amdsmi/tests/rocm_smi_test/main.cc b/projects/amdsmi/tests/rocm_smi_test/main.cc index c8ca0171bb..4ad0fe8ffc 100755 --- a/projects/amdsmi/tests/rocm_smi_test/main.cc +++ b/projects/amdsmi/tests/rocm_smi_test/main.cc @@ -78,6 +78,7 @@ #include "functional/mem_page_info_read.h" #include "functional/api_support_read.h" #include "functional/mutual_exclusion.h" +#include "functional/evt_notif_read_write.h" static RSMITstGlobals *sRSMIGlvalues = nullptr; @@ -232,6 +233,10 @@ TEST(rsmitstReadOnly, TestMutualExclusion) { test.Run(); RunCustomTestEpilog(&test); } +TEST(rsmitstReadWrite, TestEvtNotifReadWrite) { + TestEvtNotifReadWrite tst; + RunGenericTest(&tst); +} int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); diff --git a/projects/amdsmi/tests/rocm_smi_test/test_utils.cc b/projects/amdsmi/tests/rocm_smi_test/test_utils.cc index 40add62a83..8eda7f19e8 100755 --- a/projects/amdsmi/tests/rocm_smi_test/test_utils.cc +++ b/projects/amdsmi/tests/rocm_smi_test/test_utils.cc @@ -72,8 +72,15 @@ static const std::map kDevFWNameMap = { {RSMI_FW_BLOCK_VCN, "vcn"}, }; - const char * NameFromFWEnum(rsmi_fw_block_t blk) { return kDevFWNameMap.at(blk); } + +static const std::map kEvtNotifEvntNameMap = { + {RSMI_EVT_NOTIF_VMFAULT, "RSMI_EVT_NOTIF_VMFAULT"}, +}; +const char * +NameFromEvtNotifType(rsmi_evt_notification_type_t evt) { + return kEvtNotifEvntNameMap.at(evt); +} diff --git a/projects/amdsmi/tests/rocm_smi_test/test_utils.h b/projects/amdsmi/tests/rocm_smi_test/test_utils.h index 8806a824b0..28185a1307 100755 --- a/projects/amdsmi/tests/rocm_smi_test/test_utils.h +++ b/projects/amdsmi/tests/rocm_smi_test/test_utils.h @@ -50,5 +50,7 @@ const char * NameFromFWEnum(rsmi_fw_block_t blk); +const char * +NameFromEvtNotifType(rsmi_evt_notification_type_t evt); #endif // TESTS_ROCM_SMI_TEST_TEST_UTILS_H_