AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

libhsakmt: Prefix global symbols with hsakmt

Browse Source 
To support fully-static library ROCm builds, ensure that all global
symbols are prefixed with something meaningful to avoid collisions with
other libraries

A script was made using" objdump -C -t" to get a list of symbols,
then checking if the global symbols have a meaningful prefix (for thunk:
hsakmt or kmt in various cases)

Change-Id: Ifd353f64a3344eb60d1f6c4e041aa20967b38a59
Signed-off-by: Kent Russell <kent.russell@amd.com>


[ROCm/ROCR-Runtime commit: 3da42a0847]
This commit is contained in:
Kent Russell
2024-08-23 14:15:16 -04:00
parent f0baa90369
commit 9d797771b5
37 changed files with 484 additions and 484 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rocr-runtime/libhsakmt/src/debug.c
+22 -22
View File
@@ -33,7 +33,7 @@
static bool *is_device_debugged;
static uint32_t runtime_capabilities_mask = 0;
HSAKMT_STATUS init_device_debugging_memory(unsigned int NumNodes)
HSAKMT_STATUS hsakmt_init_device_debugging_memory(unsigned int NumNodes)
{
unsigned int i;
@@ -47,7 +47,7 @@ HSAKMT_STATUS init_device_debugging_memory(unsigned int NumNodes)
return HSAKMT_STATUS_SUCCESS;
}
void destroy_device_debugging_memory(void)
void hsakmt_destroy_device_debugging_memory(void)
{
if (is_device_debugged) {
free(is_device_debugged);
@@ -55,7 +55,7 @@ void destroy_device_debugging_memory(void)
}
}
bool debug_get_reg_status(uint32_t node_id)
bool hsakmt_debug_get_reg_status(uint32_t node_id)
{
return is_device_debugged[node_id];
}
@@ -70,7 +70,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDbgRegister(HSAuint32 NodeId)
if (!is_device_debugged)
return HSAKMT_STATUS_NO_MEMORY;
result = validate_nodeid(NodeId, &gpu_id);
result = hsakmt_validate_nodeid(NodeId, &gpu_id);
if (result != HSAKMT_STATUS_SUCCESS)
return result;
@@ -78,7 +78,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDbgRegister(HSAuint32 NodeId)
args.gpu_id = gpu_id;
long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_REGISTER_DEPRECATED, &args);
long err = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_DBG_REGISTER_DEPRECATED, &args);
if (err == 0)
result = HSAKMT_STATUS_SUCCESS;
@@ -98,14 +98,14 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDbgUnregister(HSAuint32 NodeId)
if (!is_device_debugged)
return HSAKMT_STATUS_NO_MEMORY;
result = validate_nodeid(NodeId, &gpu_id);
result = hsakmt_validate_nodeid(NodeId, &gpu_id);
if (result != HSAKMT_STATUS_SUCCESS)
return result;
struct kfd_ioctl_dbg_unregister_args args = {0};
args.gpu_id = gpu_id;
long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_UNREGISTER_DEPRECATED, &args);
long err = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_DBG_UNREGISTER_DEPRECATED, &args);
if (err)
return HSAKMT_STATUS_ERROR;
@@ -126,7 +126,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDbgWavefrontControl(HSAuint32 NodeId,
CHECK_KFD_OPEN();
result = validate_nodeid(NodeId, &gpu_id);
result = hsakmt_validate_nodeid(NodeId, &gpu_id);
if (result != HSAKMT_STATUS_SUCCESS)
return result;
@@ -168,7 +168,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDbgWavefrontControl(HSAuint32 NodeId,
run_ptr += sizeof(DbgWaveMsgRing->MemoryVA);
/* send to kernel */
long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_WAVE_CONTROL_DEPRECATED, args);
long err = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_DBG_WAVE_CONTROL_DEPRECATED, args);
free(args);
@@ -199,7 +199,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDbgAddressWatch(HSAuint32 NodeId,
CHECK_KFD_OPEN();
result = validate_nodeid(NodeId, &gpu_id);
result = hsakmt_validate_nodeid(NodeId, &gpu_id);
if (result != HSAKMT_STATUS_SUCCESS)
return result;
@@ -256,7 +256,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDbgAddressWatch(HSAuint32 NodeId,
}
/* send to kernel */
long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_ADDRESS_WATCH_DEPRECATED, args);
long err = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_DBG_ADDRESS_WATCH_DEPRECATED, args);
free(args);
@@ -316,7 +316,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRuntimeEnable(void *rDebug,
((setupTtmp) ? KFD_RUNTIME_ENABLE_MODE_TTMP_SAVE_MASK : 0);
args.r_debug = (HSAuint64)rDebug;
long err = kmtIoctl(kfd_fd, AMDKFD_IOC_RUNTIME_ENABLE, &args);
long err = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_RUNTIME_ENABLE, &args);
if (err) {
if (errno == EBUSY)
@@ -340,7 +340,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRuntimeDisable(void)
memset(&args, 0x00, sizeof(args));
args.mode_mask = 0; //Disable
if (kmtIoctl(kfd_fd, AMDKFD_IOC_RUNTIME_ENABLE, &args))
if (hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_RUNTIME_ENABLE, &args))
return HSAKMT_STATUS_ERROR;
return HSAKMT_STATUS_SUCCESS;
@@ -363,7 +363,7 @@ static HSAKMT_STATUS dbg_trap_get_device_data(void *data,
args.device_snapshot.entry_size = entry_size;
args.op = KFD_IOC_DBG_TRAP_GET_DEVICE_SNAPSHOT;
args.pid = getpid();
if (kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, &args))
if (hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_DBG_TRAP, &args))
return HSAKMT_STATUS_ERROR;
*n_entries = args.device_snapshot.num_devices;
@@ -384,7 +384,7 @@ static HSAKMT_STATUS dbg_trap_get_queue_data(void *data,
args.queue_snapshot.snapshot_buf_ptr = (uint64_t) data;
args.pid = getpid();
if (kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, &args))
if (hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_DBG_TRAP, &args))
return HSAKMT_STATUS_ERROR;
*n_entries = args.queue_snapshot.num_queues;
@@ -410,7 +410,7 @@ static HSAKMT_STATUS dbg_trap_suspend_queues(uint32_t *queue_ids,
args.op = KFD_IOC_DBG_TRAP_SUSPEND_QUEUES;
args.pid = getpid();
r = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, &args);
r = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_DBG_TRAP, &args);
if (r < 0)
return HSAKMT_STATUS_ERROR;
@@ -429,7 +429,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDbgEnable(void **runtime_info,
CHECK_KFD_MINOR_VERSION(KFD_MINOR_MIN_DEBUG);
*data_size = sizeof(struct kfd_runtime_info);
args.enable.rinfo_size = *data_size;
args.enable.dbg_fd = kfd_fd;
args.enable.dbg_fd = hsakmt_kfd_fd;
*runtime_info = malloc(args.enable.rinfo_size);
if (!*runtime_info)
return HSAKMT_STATUS_NO_MEMORY;
@@ -437,7 +437,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDbgEnable(void **runtime_info,
args.op = KFD_IOC_DBG_TRAP_ENABLE;
args.pid = getpid();
if (kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, &args)) {
if (hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_DBG_TRAP, &args)) {
free(*runtime_info);
return HSAKMT_STATUS_ERROR;
}
@@ -450,11 +450,11 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDbgDisable(void)
CHECK_KFD_OPEN();
CHECK_KFD_MINOR_VERSION(KFD_MINOR_MIN_DEBUG);
args.enable.dbg_fd = kfd_fd;
args.enable.dbg_fd = hsakmt_kfd_fd;
args.op = KFD_IOC_DBG_TRAP_DISABLE;
args.pid = getpid();
if (kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, &args))
if (hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_DBG_TRAP, &args))
return HSAKMT_STATUS_ERROR;
return HSAKMT_STATUS_SUCCESS;
@@ -532,11 +532,11 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDebugTrapIoctl(struct kfd_ioctl_dbg_trap_args *arg
(void *)args->suspend_queues.queue_array_ptr :
(void *)args->resume_queues.queue_array_ptr;
memcpy(queue_ptr, convert_queue_ids(num_queues, Queues),
memcpy(queue_ptr, hsakmt_convert_queue_ids(num_queues, Queues),
num_queues * sizeof(uint32_t));
}
long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, args);
long err = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_DBG_TRAP, args);
if (DebugReturn)
*DebugReturn = err;
projects/rocr-runtime/libhsakmt/src/events.c
+20 -20
View File
@@ -36,7 +36,7 @@
static HSAuint64 *events_page = NULL;
void clear_events_page(void)
void hsakmt_clear_events_page(void)
{
events_page = NULL;
}
@@ -74,18 +74,18 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateEvent(HsaEventDescriptor *EventDesc,
/* dGPU code */
pthread_mutex_lock(&hsakmt_mutex);
if (is_dgpu && !events_page) {
events_page = allocate_exec_aligned_memory_gpu(
if (hsakmt_is_dgpu && !events_page) {
events_page = hsakmt_allocate_exec_aligned_memory_gpu(
KFD_SIGNAL_EVENT_LIMIT * 8, PAGE_SIZE, 0, 0, true, false, true);
if (!events_page) {
free(e);
pthread_mutex_unlock(&hsakmt_mutex);
return HSAKMT_STATUS_ERROR;
}
fmm_get_handle(events_page, (uint64_t *)&args.event_page_offset);
hsakmt_fmm_get_handle(events_page, (uint64_t *)&args.event_page_offset);
}
if (kmtIoctl(kfd_fd, AMDKFD_IOC_CREATE_EVENT, &args) != 0) {
if (hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_CREATE_EVENT, &args) != 0) {
free(e);
*Event = NULL;
pthread_mutex_unlock(&hsakmt_mutex);
@@ -96,12 +96,12 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateEvent(HsaEventDescriptor *EventDesc,
if (!events_page && args.event_page_offset > 0) {
events_page = mmap(NULL, event_limit * 8, PROT_WRITE | PROT_READ,
MAP_SHARED, kfd_fd, args.event_page_offset);
MAP_SHARED, hsakmt_kfd_fd, args.event_page_offset);
if (events_page == MAP_FAILED) {
/* old kernels only support 256 events */
event_limit = 256;
events_page = mmap(NULL, PAGE_SIZE, PROT_WRITE | PROT_READ,
MAP_SHARED, kfd_fd, args.event_page_offset);
MAP_SHARED, hsakmt_kfd_fd, args.event_page_offset);
}
if (events_page == MAP_FAILED) {
events_page = NULL;
@@ -130,7 +130,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateEvent(HsaEventDescriptor *EventDesc,
set_args.event_id = args.event_id;
kmtIoctl(kfd_fd, AMDKFD_IOC_SET_EVENT, &set_args);
hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_SET_EVENT, &set_args);
}
*Event = e;
@@ -149,7 +149,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDestroyEvent(HsaEvent *Event)
args.event_id = Event->EventId;
if (kmtIoctl(kfd_fd, AMDKFD_IOC_DESTROY_EVENT, &args) != 0)
if (hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_DESTROY_EVENT, &args) != 0)
return HSAKMT_STATUS_ERROR;
free(Event);
@@ -173,7 +173,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetEvent(HsaEvent *Event)
args.event_id = Event->EventId;
if (kmtIoctl(kfd_fd, AMDKFD_IOC_SET_EVENT, &args) == -1)
if (hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_SET_EVENT, &args) == -1)
return HSAKMT_STATUS_ERROR;
return HSAKMT_STATUS_SUCCESS;
@@ -196,7 +196,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtResetEvent(HsaEvent *Event)
args.event_id = Event->EventId;
if (kmtIoctl(kfd_fd, AMDKFD_IOC_RESET_EVENT, &args) == -1)
if (hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_RESET_EVENT, &args) == -1)
return HSAKMT_STATUS_ERROR;
return HSAKMT_STATUS_SUCCESS;
@@ -250,7 +250,7 @@ static HSAKMT_STATUS get_mem_info_svm_api(uint64_t address, uint32_t gpu_id)
args->op = KFD_IOCTL_SVM_OP_GET_ATTR;
args->nattr = s_attr / sizeof(*attrs);
memcpy(args->attrs, attrs, s_attr);
if (kmtIoctl(kfd_fd, AMDKFD_IOC_SVM + (s_attr << _IOC_SIZESHIFT), args)) {
if (hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_SVM + (s_attr << _IOC_SIZESHIFT), args)) {
pr_debug("op get range attrs failed %s\n", strerror(errno));
return HSAKMT_STATUS_ERROR;
}
@@ -261,7 +261,7 @@ static HSAKMT_STATUS get_mem_info_svm_api(uint64_t address, uint32_t gpu_id)
args->attrs[i].value == KFD_IOCTL_SVM_LOCATION_UNDEFINED)
node_id = args->attrs[i].value;
else
gpuid_to_nodeid(args->attrs[i].value, &node_id);
hsakmt_gpuid_to_nodeid(args->attrs[i].value, &node_id);
switch (args->attrs[i].type) {
case KFD_IOCTL_SVM_ATTR_PREFERRED_LOC:
pr_err("Preferred location for address 0x%lx is Node id %d\n",
@@ -313,7 +313,7 @@ static void analysis_memory_exception(struct kfd_hsa_memory_exception_data *
uint32_t node_id = 0;
unsigned int i;
gpuid_to_nodeid(memory_exception_data->gpu_id, &node_id);
hsakmt_gpuid_to_nodeid(memory_exception_data->gpu_id, &node_id);
pr_err("Memory exception on virtual address 0x%lx, ", addr);
pr_err("node id %d : ", node_id);
if (memory_exception_data->failure.NotPresent)
@@ -323,7 +323,7 @@ static void analysis_memory_exception(struct kfd_hsa_memory_exception_data *
else if (memory_exception_data->failure.NoExecute)
pr_err("Execute to none-executable page\n");
ret = fmm_get_mem_info((const void *)addr, &info);
ret = hsakmt_fmm_get_mem_info((const void *)addr, &info);
if (ret != HSAKMT_STATUS_SUCCESS) {
ret = get_mem_info_svm_api(addr, memory_exception_data->gpu_id);
if (ret != HSAKMT_STATUS_SUCCESS)
@@ -407,7 +407,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnMultipleEvents_Ext(HsaEvent *Events[],
HSAKMT_STATUS result;
if (kmtIoctl(kfd_fd, AMDKFD_IOC_WAIT_EVENTS, &args) == -1)
if (hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_WAIT_EVENTS, &args) == -1)
result = HSAKMT_STATUS_ERROR;
else if (args.wait_result == KFD_IOC_WAIT_RESULT_TIMEOUT)
result = HSAKMT_STATUS_WAIT_TIMEOUT;
@@ -417,7 +417,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnMultipleEvents_Ext(HsaEvent *Events[],
if (Events[i]->EventData.EventType == HSA_EVENTTYPE_MEMORY &&
event_data[i].memory_exception_data.gpu_id) {
Events[i]->EventData.EventData.MemoryAccessFault.VirtualAddress = event_data[i].memory_exception_data.va;
result = gpuid_to_nodeid(event_data[i].memory_exception_data.gpu_id, &Events[i]->EventData.EventData.MemoryAccessFault.NodeId);
result = hsakmt_gpuid_to_nodeid(event_data[i].memory_exception_data.gpu_id, &Events[i]->EventData.EventData.MemoryAccessFault.NodeId);
if (result != HSAKMT_STATUS_SUCCESS)
goto out;
Events[i]->EventData.EventData.MemoryAccessFault.Failure.NotPresent = event_data[i].memory_exception_data.failure.NotPresent;
@@ -432,7 +432,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnMultipleEvents_Ext(HsaEvent *Events[],
} else if (Events[i]->EventData.EventType == HSA_EVENTTYPE_HW_EXCEPTION &&
event_data[i].hw_exception_data.gpu_id) {
result = gpuid_to_nodeid(event_data[i].hw_exception_data.gpu_id, &Events[i]->EventData.EventData.HwException.NodeId);
result = hsakmt_gpuid_to_nodeid(event_data[i].hw_exception_data.gpu_id, &Events[i]->EventData.EventData.HwException.NodeId);
if (result != HSAKMT_STATUS_SUCCESS)
goto out;
@@ -464,14 +464,14 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenSMI(HSAuint32 NodeId, int *fd)
pr_debug("[%s] node %d\n", __func__, NodeId);
result = validate_nodeid(NodeId, &gpuid);
result = hsakmt_validate_nodeid(NodeId, &gpuid);
if (result != HSAKMT_STATUS_SUCCESS) {
pr_err("[%s] invalid node ID: %d\n", __func__, NodeId);
return result;
}
args.gpuid = gpuid;
result = kmtIoctl(kfd_fd, AMDKFD_IOC_SMI_EVENTS, &args);
result = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_SMI_EVENTS, &args);
if (result) {
pr_debug("open SMI event fd failed %s\n", strerror(errno));
return HSAKMT_STATUS_ERROR;
projects/rocr-runtime/libhsakmt/src/fmm.c
+116 -116
View File
@@ -399,9 +399,9 @@ static void vm_remove_object(manageable_aperture_t *app, vm_object_t *object)
if (object->mapped_node_id_array)
free(object->mapped_node_id_array);
rbtree_delete(&app->tree, &object->node);
hsakmt_rbtree_delete(&app->tree, &object->node);
if (object->userptr)
rbtree_delete(&app->user_tree, &object->user_node);
hsakmt_rbtree_delete(&app->user_tree, &object->user_node);
free(object);
}
@@ -532,7 +532,7 @@ static vm_object_t *vm_find_object_by_address_userptr_range(manageable_aperture_
if (ln == rn)
break;
rn = rbtree_prev(tree, rn);
rn = hsakmt_rbtree_prev(tree, rn);
}
return cur; /* NULL if not found */
@@ -730,7 +730,7 @@ static void *reserved_aperture_allocate_aligned(manageable_aperture_t *app,
return start;
}
void *mmap_allocate_aligned(int prot, int flags, uint64_t size, uint64_t align,
void *hsakmt_mmap_allocate_aligned(int prot, int flags, uint64_t size, uint64_t align,
uint64_t guard_size, void *aper_base, void *aper_limit)
{
void *addr, *aligned_addr, *aligned_end, *mapping_end;
@@ -831,7 +831,7 @@ static void *mmap_aperture_allocate_aligned(manageable_aperture_t *aper,
*/
guard_size = (uint64_t)aper->guard_pages * PAGE_SIZE;
return mmap_allocate_aligned(PROT_NONE, MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE,
return hsakmt_mmap_allocate_aligned(PROT_NONE, MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE,
size, align, guard_size, aper->base, aper->limit);
}
@@ -885,7 +885,7 @@ static vm_object_t *aperture_allocate_object(manageable_aperture_t *app,
if (!new_object)
return NULL;
rbtree_insert(&app->tree, &new_object->node);
hsakmt_rbtree_insert(&app->tree, &new_object->node);
return new_object;
}
@@ -959,7 +959,7 @@ static manageable_aperture_t *fmm_find_aperture(const void *address,
aperture = &mem_handle_aperture;
_info.type = HSA_APERTURE_MEMHANDLE;
} else if (is_dgpu) {
} else if (hsakmt_is_dgpu) {
if (address >= svm.dgpu_aperture->base &&
address <= svm.dgpu_aperture->limit) {
@@ -1048,7 +1048,7 @@ static HSAKMT_STATUS fmm_register_mem_svm_api(void *address,
pr_debug("Registering to SVM %p size: %ld\n", (void*)aligned_addr,
aligned_size);
/* Driver does one copy_from_user, with extra attrs size */
if (kmtIoctl(kfd_fd, AMDKFD_IOC_SVM + (s_attr << _IOC_SIZESHIFT), args)) {
if (hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_SVM + (s_attr << _IOC_SIZESHIFT), args)) {
pr_debug("op set range attrs failed %s\n", strerror(errno));
return HSAKMT_STATUS_ERROR;
}
@@ -1081,7 +1081,7 @@ static HSAKMT_STATUS fmm_map_mem_svm_api(void *address,
args->attrs[i].value = nodes_to_map[i];
}
/* Driver does one copy_from_user, with extra attrs size */
if (kmtIoctl(kfd_fd, AMDKFD_IOC_SVM + (s_attr << _IOC_SIZESHIFT), args)) {
if (hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_SVM + (s_attr << _IOC_SIZESHIFT), args)) {
pr_debug("op set range attrs failed %s\n", strerror(errno));
return HSAKMT_STATUS_ERROR;
}
@@ -1113,7 +1113,7 @@ static vm_object_t *fmm_allocate_memory_object(uint32_t gpu_id, void *mem,
args.flags = ioc_flags |
KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE;
args.va_addr = (uint64_t)mem;
if (!is_dgpu &&
if (!hsakmt_is_dgpu &&
(ioc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM))
args.va_addr = VOID_PTRS_SUB(mem, aperture->base);
if (ioc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR)
@@ -1123,7 +1123,7 @@ static vm_object_t *fmm_allocate_memory_object(uint32_t gpu_id, void *mem,
if (aperture == &mem_handle_aperture)
args.va_addr = 0;
if (kmtIoctl(kfd_fd, AMDKFD_IOC_ALLOC_MEMORY_OF_GPU, &args))
if (hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_ALLOC_MEMORY_OF_GPU, &args))
return NULL;
mflags = fmm_translate_ioc_to_hsa_flags(ioc_flags);
@@ -1144,7 +1144,7 @@ static vm_object_t *fmm_allocate_memory_object(uint32_t gpu_id, void *mem,
err_object_allocation_failed:
pthread_mutex_unlock(&aperture->fmm_mutex);
free_args.handle = args.handle;
kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &free_args);
hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &free_args);
return NULL;
}
@@ -1174,11 +1174,11 @@ static void manageable_aperture_print(manageable_aperture_t *app)
object = vm_object_entry(n, 0);
pr_info("\t\t Object [%p - %" PRIu64 "]\n",
object->start, object->size);
n = rbtree_next(&app->tree, n);
n = hsakmt_rbtree_next(&app->tree, n);
}
}
void fmm_print(uint32_t gpu_id)
void hsakmt_fmm_print(uint32_t gpu_id)
{
int32_t gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id);
@@ -1202,7 +1202,7 @@ void fmm_print(uint32_t gpu_id)
manageable_aperture_print(svm.dgpu_alt_aperture);
}
#else
void fmm_print(uint32_t gpu_id)
void hsakmt_fmm_print(uint32_t gpu_id)
{
}
#endif
@@ -1289,7 +1289,7 @@ static vm_object_t *vm_find_object(const void *addr, uint64_t size,
}
no_svm:
if (!obj && !is_dgpu) {
if (!obj && !hsakmt_is_dgpu) {
/* On APUs try finding it in the CPUVM aperture */
if (aper)
pthread_mutex_unlock(&aper->fmm_mutex);
@@ -1345,7 +1345,7 @@ static void fmm_release_scratch(uint32_t gpu_id)
size = VOID_PTRS_SUB(aperture->limit, aperture->base) + 1;
if (is_dgpu) {
if (hsakmt_is_dgpu) {
/* unmap and remove all remaining objects */
pthread_mutex_lock(&aperture->fmm_mutex);
while ((n = rbtree_node_any(&aperture->tree, MID))) {
@@ -1391,7 +1391,7 @@ static uint32_t fmm_translate_hsa_to_ioc_flags(HsaMemFlags flags)
}
#define SCRATCH_ALIGN 0x10000
void *fmm_allocate_scratch(uint32_t gpu_id, void *address, uint64_t MemorySizeInBytes)
void *hsakmt_fmm_allocate_scratch(uint32_t gpu_id, void *address, uint64_t MemorySizeInBytes)
{
manageable_aperture_t *aperture_phy;
struct kfd_ioctl_set_scratch_backing_va_args args = {0};
@@ -1410,7 +1410,7 @@ void *fmm_allocate_scratch(uint32_t gpu_id, void *address, uint64_t MemorySizeIn
return NULL;
/* Allocate address space for scratch backing, 64KB aligned */
if (is_dgpu) {
if (hsakmt_is_dgpu) {
pthread_mutex_lock(&svm.dgpu_aperture->fmm_mutex);
mem = aperture_allocate_area_aligned(
svm.dgpu_aperture, address,
@@ -1420,7 +1420,7 @@ void *fmm_allocate_scratch(uint32_t gpu_id, void *address, uint64_t MemorySizeIn
if (address)
return NULL;
mem = mmap_allocate_aligned(PROT_READ | PROT_WRITE,
mem = hsakmt_mmap_allocate_aligned(PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS,
aligned_size, SCRATCH_ALIGN, 0,
0, (void *)LONG_MAX);
@@ -1435,7 +1435,7 @@ void *fmm_allocate_scratch(uint32_t gpu_id, void *address, uint64_t MemorySizeIn
args.gpu_id = gpu_id;
args.va_addr = ((uint64_t)mem) >> 16;
if (kmtIoctl(kfd_fd, AMDKFD_IOC_SET_SCRATCH_BACKING_VA, &args)) {
if (hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_SET_SCRATCH_BACKING_VA, &args)) {
fmm_release_scratch(gpu_id);
return NULL;
}
@@ -1529,7 +1529,7 @@ static void *fmm_allocate_va(uint32_t gpu_id, void *address, uint64_t size,
return mem;
}
void *fmm_allocate_device(uint32_t gpu_id, uint32_t node_id, void *address,
void *hsakmt_fmm_allocate_device(uint32_t gpu_id, uint32_t node_id, void *address,
uint64_t MemorySizeInBytes, uint64_t alignment, HsaMemFlags mflags)
{
manageable_aperture_t *aperture;
@@ -1551,7 +1551,7 @@ void *fmm_allocate_device(uint32_t gpu_id, uint32_t node_id, void *address,
ioc_flags |= fmm_translate_hsa_to_ioc_flags(mflags);
if (topology_is_svm_needed(gpu_mem[gpu_mem_id].EngineId)) {
if (hsakmt_topology_is_svm_needed(gpu_mem[gpu_mem_id].EngineId)) {
aperture = svm.dgpu_aperture;
if (mflags.ui32.AQLQueueMemory)
size = MemorySizeInBytes * 2;
@@ -1586,7 +1586,7 @@ void *fmm_allocate_device(uint32_t gpu_id, uint32_t node_id, void *address,
pthread_mutex_lock(&aperture->fmm_mutex);
/* Store memory allocation flags, not ioc flags */
vm_obj->mflags = mflags;
gpuid_to_nodeid(gpu_id, &vm_obj->node_id);
hsakmt_gpuid_to_nodeid(gpu_id, &vm_obj->node_id);
pthread_mutex_unlock(&aperture->fmm_mutex);
}
@@ -1613,7 +1613,7 @@ void *fmm_allocate_device(uint32_t gpu_id, uint32_t node_id, void *address,
return mem;
}
void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes,
void *hsakmt_fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes,
uint64_t doorbell_mmap_offset)
{
manageable_aperture_t *aperture;
@@ -1646,14 +1646,14 @@ void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes,
pthread_mutex_lock(&aperture->fmm_mutex);
vm_obj->mflags = mflags;
gpuid_to_nodeid(gpu_id, &vm_obj->node_id);
hsakmt_gpuid_to_nodeid(gpu_id, &vm_obj->node_id);
pthread_mutex_unlock(&aperture->fmm_mutex);
}
if (mem) {
void *ret = mmap(mem, MemorySizeInBytes,
PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_FIXED, kfd_fd,
MAP_SHARED | MAP_FIXED, hsakmt_kfd_fd,
doorbell_mmap_offset);
if (ret == MAP_FAILED) {
__fmm_release(vm_obj, aperture);
@@ -1892,10 +1892,10 @@ out_release_area:
return NULL;
}
void *fmm_allocate_host(uint32_t gpu_id, uint32_t node_id, void *address,
void *hsakmt_fmm_allocate_host(uint32_t gpu_id, uint32_t node_id, void *address,
uint64_t MemorySizeInBytes, uint64_t alignment, HsaMemFlags mflags)
{
if (is_dgpu)
if (hsakmt_is_dgpu)
return fmm_allocate_host_gpu(gpu_id, node_id, address, MemorySizeInBytes, alignment, mflags);
if (alignment) {//Alignment not supported on non-dgpu
@@ -1929,7 +1929,7 @@ static int __fmm_release(vm_object_t *object, manageable_aperture_t *aperture)
* free the BO before unmapping the pages.
*/
args.handle = object->handle;
if (args.handle && kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &args)) {
if (args.handle && hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &args)) {
pthread_mutex_unlock(&aperture->fmm_mutex);
return -errno;
}
@@ -1941,7 +1941,7 @@ static int __fmm_release(vm_object_t *object, manageable_aperture_t *aperture)
return 0;
}
HSAKMT_STATUS fmm_release(void *address)
HSAKMT_STATUS hsakmt_fmm_release(void *address)
{
manageable_aperture_t *aperture = NULL;
vm_object_t *object = NULL;
@@ -1959,7 +1959,7 @@ HSAKMT_STATUS fmm_release(void *address)
object = vm_find_object(address, 0, &aperture);
if (!object)
return is_svm_api_supported ?
return hsakmt_is_svm_api_supported ?
HSAKMT_STATUS_SUCCESS :
HSAKMT_STATUS_MEMORY_NOT_REGISTERED;
@@ -1978,7 +1978,7 @@ HSAKMT_STATUS fmm_release(void *address)
return HSAKMT_STATUS_ERROR;
if (!aperture->is_cpu_accessible)
fmm_print(gpu_mem[i].gpu_id);
hsakmt_fmm_print(gpu_mem[i].gpu_id);
}
return HSAKMT_STATUS_SUCCESS;
@@ -1995,7 +1995,7 @@ static int fmm_set_memory_policy(uint32_t gpu_id, int default_policy, int alt_po
args.alternate_aperture_base = alt_base;
args.alternate_aperture_size = alt_size;
return kmtIoctl(kfd_fd, AMDKFD_IOC_SET_MEMORY_POLICY, &args);
return hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_SET_MEMORY_POLICY, &args);
}
static uint32_t get_vm_alignment(uint32_t device_id)
@@ -2019,7 +2019,7 @@ static HSAKMT_STATUS get_process_apertures(
args_new.kfd_process_device_apertures_ptr = (uintptr_t)process_apertures;
args_new.num_of_nodes = *num_of_nodes;
if (!kmtIoctl(kfd_fd, AMDKFD_IOC_GET_PROCESS_APERTURES_NEW,
if (!hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_GET_PROCESS_APERTURES_NEW,
(void *)&args_new)) {
*num_of_nodes = args_new.num_of_nodes;
return HSAKMT_STATUS_SUCCESS;
@@ -2029,7 +2029,7 @@ static HSAKMT_STATUS get_process_apertures(
* a really old kernel */
memset(&args_old, 0, sizeof(args_old));
if (kmtIoctl(kfd_fd, AMDKFD_IOC_GET_PROCESS_APERTURES,
if (hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_GET_PROCESS_APERTURES,
(void *)&args_old))
return HSAKMT_STATUS_ERROR;
@@ -2054,7 +2054,7 @@ static int drm_render_fds[DRM_LAST_RENDER_NODE + 1 - DRM_FIRST_RENDER_NODE];
/* amdgpu device handle for each gpu that libdrm uses */
static struct amdgpu_device *amdgpu_handle[DRM_LAST_RENDER_NODE + 1 - DRM_FIRST_RENDER_NODE];
int open_drm_render_device(int minor)
int hsakmt_open_drm_render_device(int minor)
{
char path[128];
int index, fd;
@@ -2089,8 +2089,8 @@ int open_drm_render_device(int minor)
/* if amdgpu_device_get_fd available query render fd that libdrm uses,
* then close drm_render_fds above, replace it by fd libdrm uses.
*/
if (fn_amdgpu_device_get_fd) {
fd = fn_amdgpu_device_get_fd(*device_handle);
if (hsakmt_fn_amdgpu_device_get_fd) {
fd = hsakmt_fn_amdgpu_device_get_fd(*device_handle);
if (fd > 0) {
close(drm_render_fds[index]);
drm_render_fds[index] = fd;
@@ -2112,7 +2112,7 @@ static HSAKMT_STATUS acquire_vm(uint32_t gpu_id, int fd)
args.gpu_id = gpu_id;
args.drm_fd = fd;
pr_info("acquiring VM for %x using %d\n", gpu_id, fd);
if (kmtIoctl(kfd_fd, AMDKFD_IOC_ACQUIRE_VM, (void *)&args)) {
if (hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_ACQUIRE_VM, (void *)&args)) {
pr_err("AMDKFD_IOC_ACQUIRE_VM failed\n");
return HSAKMT_STATUS_ERROR;
}
@@ -2392,7 +2392,7 @@ static void *map_mmio(uint32_t node_id, uint32_t gpu_id, int mmap_fd)
}
/* Map for GPU access*/
if (fmm_map_to_gpu(mem, PAGE_SIZE, NULL)) {
if (hsakmt_fmm_map_to_gpu(mem, PAGE_SIZE, NULL)) {
__fmm_release(vm_obj, aperture);
return NULL;
}
@@ -2407,13 +2407,13 @@ static void release_mmio(void)
for (gpu_mem_id = 0; (uint32_t)gpu_mem_id < gpu_mem_count; gpu_mem_id++) {
if (!gpu_mem[gpu_mem_id].mmio_aperture.base)
continue;
fmm_unmap_from_gpu(gpu_mem[gpu_mem_id].mmio_aperture.base);
hsakmt_fmm_unmap_from_gpu(gpu_mem[gpu_mem_id].mmio_aperture.base);
munmap(gpu_mem[gpu_mem_id].mmio_aperture.base, PAGE_SIZE);
fmm_release(gpu_mem[gpu_mem_id].mmio_aperture.base);
hsakmt_fmm_release(gpu_mem[gpu_mem_id].mmio_aperture.base);
}
}
HSAKMT_STATUS fmm_get_amdgpu_device_handle(uint32_t node_id,
HSAKMT_STATUS hsakmt_fmm_get_amdgpu_device_handle(uint32_t node_id,
HsaAMDGPUDeviceHandle *DeviceHandle)
{
int32_t i = gpu_mem_find_by_node_id(node_id);
@@ -2491,7 +2491,7 @@ static bool init_mem_handle_aperture(HSAuint32 align, HSAuint32 guard_pages)
return false;
}
HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes)
HSAKMT_STATUS hsakmt_fmm_init_process_apertures(unsigned int NumNodes)
{
uint32_t i;
int32_t gpu_mem_id = 0;
@@ -2554,20 +2554,20 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes)
* gets called before hsaKmtAcquireSystemProperties() is called.
*/
is_dgpu = false;
hsakmt_is_dgpu = false;
for (i = 0; i < NumNodes; i++) {
HsaNodeProperties props;
ret = topology_get_node_props(i, &props);
ret = hsakmt_topology_get_node_props(i, &props);
if (ret != HSAKMT_STATUS_SUCCESS)
goto gpu_mem_init_failed;
topology_setup_is_dgpu_param(&props);
hsakmt_topology_setup_is_dgpu_param(&props);
/* Skip non-GPU nodes */
if (props.KFDGpuID) {
int fd = open_drm_render_device(props.DrmRenderMinor);
int fd = hsakmt_open_drm_render_device(props.DrmRenderMinor);
if (fd <= 0) {
ret = HSAKMT_STATUS_ERROR;
goto gpu_mem_init_failed;
@@ -2592,7 +2592,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes)
gpu_mem[gpu_mem_count].local_mem_size = props.LocalMemSize;
gpu_mem[gpu_mem_count].device_id = props.DeviceId;
gpu_mem[gpu_mem_count].node_id = i;
is_svm_api_supported &= props.Capability.ui32.SVMAPISupported;
hsakmt_is_svm_api_supported &= props.Capability.ui32.SVMAPISupported;
gpu_mem[gpu_mem_count].scratch_physical.align = PAGE_SIZE;
gpu_mem[gpu_mem_count].scratch_physical.ops = &reserved_aperture_ops;
@@ -2616,7 +2616,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes)
* required since Number of nodes is already known. Kernel will fill in
* the apertures in kfd_process_device_apertures_ptr
*/
num_of_sysfs_nodes = get_num_sysfs_nodes();
num_of_sysfs_nodes = hsakmt_get_num_sysfs_nodes();
if (num_of_sysfs_nodes < gpu_mem_count) {
ret = HSAKMT_STATUS_ERROR;
goto sysfs_parse_failed;
@@ -2670,11 +2670,11 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes)
* allocated on those GPUs.
*/
nodeId = gpu_mem[gpu_mem_id].node_id;
ret = topology_get_node_props(nodeId, &nodeProps);
ret = hsakmt_topology_get_node_props(nodeId, &nodeProps);
if (ret != HSAKMT_STATUS_SUCCESS)
goto aperture_init_failed;
assert(nodeProps.NumIOLinks <= NumNodes);
ret = topology_get_iolink_props(nodeId, nodeProps.NumIOLinks,
ret = hsakmt_topology_get_iolink_props(nodeId, nodeProps.NumIOLinks,
linkProps);
if (ret != HSAKMT_STATUS_SUCCESS)
goto aperture_init_failed;
@@ -2791,12 +2791,12 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes)
pr_err("Failed to init mem_handle_aperture\n");
for (gpu_mem_id = 0; (uint32_t)gpu_mem_id < gpu_mem_count; gpu_mem_id++) {
if (!topology_is_svm_needed(gpu_mem[gpu_mem_id].EngineId))
if (!hsakmt_topology_is_svm_needed(gpu_mem[gpu_mem_id].EngineId))
continue;
gpu_mem[gpu_mem_id].mmio_aperture.base = map_mmio(
gpu_mem[gpu_mem_id].node_id,
gpu_mem[gpu_mem_id].gpu_id,
kfd_fd);
hsakmt_kfd_fd);
if (gpu_mem[gpu_mem_id].mmio_aperture.base)
gpu_mem[gpu_mem_id].mmio_aperture.limit = (void *)
((char *)gpu_mem[gpu_mem_id].mmio_aperture.base +
@@ -2818,11 +2818,11 @@ get_aperture_ioctl_failed:
free(process_apertures);
sysfs_parse_failed:
gpu_mem_init_failed:
fmm_destroy_process_apertures();
hsakmt_fmm_destroy_process_apertures();
return ret;
}
void fmm_destroy_process_apertures(void)
void hsakmt_fmm_destroy_process_apertures(void)
{
release_mmio();
if (gpu_mem) {
@@ -2834,7 +2834,7 @@ void fmm_destroy_process_apertures(void)
gpu_mem_count = 0;
}
HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSAuint32 gpu_id,
HSAKMT_STATUS hsakmt_fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSAuint32 gpu_id,
HSAuint64 *aperture_base, HSAuint64 *aperture_limit)
{
HSAKMT_STATUS err = HSAKMT_STATUS_ERROR;
@@ -3011,7 +3011,7 @@ static HSAKMT_STATUS _fmm_map_to_gpu(manageable_aperture_t *aperture,
/* not specified, not registered: map all GPUs */
int32_t gpu_mem_id = gpu_mem_find_by_node_id(obj->node_id);
if (!obj->userptr && get_device_id_by_node_id(obj->node_id) &&
if (!obj->userptr && hsakmt_get_device_id_by_node_id(obj->node_id) &&
gpu_mem_id >= 0) {
args.device_ids_array_ptr = (uint64_t)
gpu_mem[gpu_mem_id].usable_peer_id_array;
@@ -3024,7 +3024,7 @@ static HSAKMT_STATUS _fmm_map_to_gpu(manageable_aperture_t *aperture,
}
args.n_success = 0;
ret_ioctl = kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU, &args);
ret_ioctl = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU, &args);
if (ret_ioctl) {
pr_err("GPU mapping failed (%d) for obj at %p, userptr %p, size %lu",
ret_ioctl, object->start, object->userptr, object->size);
@@ -3072,7 +3072,7 @@ static HSAKMT_STATUS _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_apertur
if (gpu_mem_id < 0)
return HSAKMT_STATUS_INVALID_PARAMETER;
if (!is_dgpu)
if (!hsakmt_is_dgpu)
return HSAKMT_STATUS_SUCCESS; /* Nothing to do on APU */
/* sanity check the address */
@@ -3080,7 +3080,7 @@ static HSAKMT_STATUS _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_apertur
VOID_PTR_ADD(address, size - 1) > aperture->limit)
return HSAKMT_STATUS_INVALID_PARAMETER;
is_debugger = debug_get_reg_status(gpu_mem[gpu_mem_id].node_id);
is_debugger = hsakmt_debug_get_reg_status(gpu_mem[gpu_mem_id].node_id);
flags = is_debugger ? KFD_IOC_ALLOC_MEM_FLAGS_GTT :
KFD_IOC_ALLOC_MEM_FLAGS_VRAM;
flags |= KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE;
@@ -3120,7 +3120,7 @@ static HSAKMT_STATUS _fmm_map_to_gpu_userptr(void *addr, uint64_t size,
/* Map and return the GPUVM address adjusted by the offset
* from the start of the page
*/
if (!object && is_svm_api_supported) {
if (!object && hsakmt_is_svm_api_supported) {
svm_addr = (void*)((HSAuint64)addr - page_offset);
if (!nodes_to_map) {
nodes_to_map = all_gpu_id_array;
@@ -3143,7 +3143,7 @@ static HSAKMT_STATUS _fmm_map_to_gpu_userptr(void *addr, uint64_t size,
return ret;
}
HSAKMT_STATUS fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address)
HSAKMT_STATUS hsakmt_fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address)
{
manageable_aperture_t *aperture;
vm_object_t *object;
@@ -3160,8 +3160,8 @@ HSAKMT_STATUS fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_addre
address, size);
object = vm_find_object(address, size, &aperture);
if (!object && !is_svm_api_supported) {
if (!is_dgpu) {
if (!object && !hsakmt_is_svm_api_supported) {
if (!hsakmt_is_dgpu) {
/* Prefetch memory on APUs with dummy-reads */
fmm_check_user_memory(address, size);
return HSAKMT_STATUS_SUCCESS;
@@ -3187,7 +3187,7 @@ HSAKMT_STATUS fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_addre
/* Prefetch memory on APUs with dummy-reads */
fmm_check_user_memory(address, size);
ret = HSAKMT_STATUS_SUCCESS;
} else if ((is_svm_api_supported && !object) || object->userptr) {
} else if ((hsakmt_is_svm_api_supported && !object) || object->userptr) {
ret = _fmm_map_to_gpu_userptr(address, size, gpuvm_address, object, NULL, 0);
} else {
ret = _fmm_map_to_gpu(aperture, address, size, object, NULL, 0);
@@ -3264,7 +3264,7 @@ static int _fmm_unmap_from_gpu(manageable_aperture_t *aperture, void *address,
print_device_id_array((void *)args.device_ids_array_ptr,
args.n_devices * sizeof(uint32_t));
ret = kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &args);
ret = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &args);
remove_device_ids_from_mapped_array(object,
(uint32_t *)args.device_ids_array_ptr,
@@ -3295,7 +3295,7 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id,
if (gpu_mem_id < 0)
return -1;
if (!is_dgpu)
if (!hsakmt_is_dgpu)
return 0; /* Nothing to do on APU */
pthread_mutex_lock(&aperture->fmm_mutex);
@@ -3318,7 +3318,7 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id,
args.device_ids_array_ptr = (uint64_t)object->mapped_device_id_array;
args.n_devices = object->mapped_device_id_array_size / sizeof(uint32_t);
args.n_success = 0;
ret = kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &args);
ret = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &args);
/* unmap from CPU while keeping the address space reserved */
mmap(address, object->size, PROT_NONE,
@@ -3346,7 +3346,7 @@ err:
return ret;
}
int fmm_unmap_from_gpu(void *address)
int hsakmt_fmm_unmap_from_gpu(void *address)
{
manageable_aperture_t *aperture;
vm_object_t *object;
@@ -3365,7 +3365,7 @@ int fmm_unmap_from_gpu(void *address)
object = vm_find_object(address, 0, &aperture);
if (!object)
/* On APUs GPU unmapping of system memory is a no-op */
return (!is_dgpu || is_svm_api_supported) ? 0 : -EINVAL;
return (!hsakmt_is_dgpu || hsakmt_is_svm_api_supported) ? 0 : -EINVAL;
/* Successful vm_find_object returns with the aperture locked */
if (aperture == &cpuvm_aperture)
@@ -3379,7 +3379,7 @@ int fmm_unmap_from_gpu(void *address)
return ret;
}
bool fmm_get_handle(void *address, uint64_t *handle)
bool hsakmt_fmm_get_handle(void *address, uint64_t *handle)
{
uint32_t i;
manageable_aperture_t *aperture;
@@ -3476,11 +3476,11 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr,
++exist_obj->registration_count;
} else {
obj->userptr = addr;
gpuid_to_nodeid(gpu_id, &obj->node_id);
hsakmt_gpuid_to_nodeid(gpu_id, &obj->node_id);
obj->userptr_size = size;
obj->registration_count = 1;
obj->user_node.key = rbtree_key((unsigned long)addr, size);
rbtree_insert(&aperture->user_tree, &obj->user_node);
hsakmt_rbtree_insert(&aperture->user_tree, &obj->user_node);
}
pthread_mutex_unlock(&aperture->fmm_mutex);
@@ -3492,7 +3492,7 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr,
return HSAKMT_STATUS_SUCCESS;
}
HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes,
HSAKMT_STATUS hsakmt_fmm_register_memory(void *address, uint64_t size_in_bytes,
uint32_t *gpu_id_array,
uint32_t gpu_id_array_size,
bool coarse_grain,
@@ -3510,12 +3510,12 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes,
object = vm_find_object(address, size_in_bytes, &aperture);
if (!object) {
if (!is_dgpu)
if (!hsakmt_is_dgpu)
/* System memory registration on APUs is a no-op */
return HSAKMT_STATUS_SUCCESS;
/* Register a new user ptr */
if (is_svm_api_supported) {
if (hsakmt_is_svm_api_supported) {
ret = fmm_register_mem_svm_api(address,
size_in_bytes,
coarse_grain,
@@ -3583,7 +3583,7 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes,
}
#define GRAPHICS_METADATA_DEFAULT_SIZE 64
HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle,
HSAKMT_STATUS hsakmt_fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle,
HsaGraphicsResourceInfo *GraphicsResourceInfo,
uint32_t *gpu_id_array,
uint32_t gpu_id_array_size)
@@ -3610,7 +3610,7 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle,
if (!metadata)
return HSAKMT_STATUS_NO_MEMORY;
infoArgs.metadata_ptr = (uint64_t)metadata;
r = kmtIoctl(kfd_fd, AMDKFD_IOC_GET_DMABUF_INFO, (void *)&infoArgs);
r = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_GET_DMABUF_INFO, (void *)&infoArgs);
if (r && infoArgs.metadata_size > GRAPHICS_METADATA_DEFAULT_SIZE) {
/* Try again with bigger metadata */
free(metadata);
@@ -3618,7 +3618,7 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle,
if (!metadata)
return HSAKMT_STATUS_NO_MEMORY;
infoArgs.metadata_ptr = (uint64_t)metadata;
r = kmtIoctl(kfd_fd, AMDKFD_IOC_GET_DMABUF_INFO, (void *)&infoArgs);
r = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_GET_DMABUF_INFO, (void *)&infoArgs);
}
if (r)
@@ -3632,7 +3632,7 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle,
/* import DMA buffer without VA assigned */
if (!gpu_id_array && gpu_id_array_size == 0) {
aperture = &mem_handle_aperture;
} else if (topology_is_svm_needed(gpu_mem[gpu_mem_id].EngineId)) {
} else if (hsakmt_topology_is_svm_needed(gpu_mem[gpu_mem_id].EngineId)) {
aperture = svm.dgpu_aperture;
} else {
aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture;
@@ -3655,7 +3655,7 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle,
importArgs.gpu_id = infoArgs.gpu_id;
importArgs.dmabuf_fd = GraphicsResourceHandle;
r = kmtIoctl(kfd_fd, AMDKFD_IOC_IMPORT_DMABUF, (void *)&importArgs);
r = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_IMPORT_DMABUF, (void *)&importArgs);
if (r)
goto error_release_aperture;
@@ -3668,7 +3668,7 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle,
obj->metadata = metadata;
obj->registered_device_id_array = gpu_id_array;
obj->registered_device_id_array_size = gpu_id_array_size;
gpuid_to_nodeid(infoArgs.gpu_id, &obj->node_id);
hsakmt_gpuid_to_nodeid(infoArgs.gpu_id, &obj->node_id);
}
pthread_mutex_unlock(&aperture->fmm_mutex);
if (!obj)
@@ -3678,13 +3678,13 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle,
GraphicsResourceInfo->SizeInBytes = infoArgs.size;
GraphicsResourceInfo->Metadata = (void *)(unsigned long)infoArgs.metadata_ptr;
GraphicsResourceInfo->MetadataSizeInBytes = infoArgs.metadata_size;
gpuid_to_nodeid(infoArgs.gpu_id, &GraphicsResourceInfo->NodeId);
hsakmt_gpuid_to_nodeid(infoArgs.gpu_id, &GraphicsResourceInfo->NodeId);
return HSAKMT_STATUS_SUCCESS;
error_release_buffer:
freeArgs.handle = importArgs.handle;
kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &freeArgs);
hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &freeArgs);
error_release_aperture:
aperture_release_area(aperture, mem, infoArgs.size);
error_free_metadata:
@@ -3693,7 +3693,7 @@ error_free_metadata:
return status;
}
HSAKMT_STATUS fmm_export_dma_buf_fd(void *MemoryAddress,
HSAKMT_STATUS hsakmt_fmm_export_dma_buf_fd(void *MemoryAddress,
HSAuint64 MemorySizeInBytes,
int *DMABufFd,
HSAuint64 *Offset)
@@ -3725,7 +3725,7 @@ HSAKMT_STATUS fmm_export_dma_buf_fd(void *MemoryAddress,
if (!obj)
return HSAKMT_STATUS_INVALID_PARAMETER;
r = kmtIoctl(kfd_fd, AMDKFD_IOC_EXPORT_DMABUF, (void *)&exportArgs);
r = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_EXPORT_DMABUF, (void *)&exportArgs);
if (r)
return HSAKMT_STATUS_ERROR;
@@ -3735,7 +3735,7 @@ HSAKMT_STATUS fmm_export_dma_buf_fd(void *MemoryAddress,
return HSAKMT_STATUS_SUCCESS;
}
HSAKMT_STATUS fmm_share_memory(void *MemoryAddress,
HSAKMT_STATUS hsakmt_fmm_share_memory(void *MemoryAddress,
HSAuint64 SizeInBytes,
HsaSharedMemoryHandle *SharedMemoryHandle)
{
@@ -3748,7 +3748,7 @@ HSAKMT_STATUS fmm_share_memory(void *MemoryAddress,
HsaSharedMemoryStruct *SharedMemoryStruct =
to_hsa_shared_memory_struct(SharedMemoryHandle);
if (SizeInBytes >= (1ULL << ((sizeof(HSAuint32) * 8) + PAGE_SHIFT)))
if (SizeInBytes >= (1ULL << ((sizeof(HSAuint32) * 8) + HSAKMT_PAGE_SHIFT)))
return HSAKMT_STATUS_INVALID_PARAMETER;
aperture = fmm_find_aperture(MemoryAddress, &ApeInfo);
@@ -3761,10 +3761,10 @@ HSAKMT_STATUS fmm_share_memory(void *MemoryAddress,
if (!obj)
return HSAKMT_STATUS_INVALID_PARAMETER;
r = validate_nodeid(obj->node_id, &gpu_id);
r = hsakmt_validate_nodeid(obj->node_id, &gpu_id);
if (r != HSAKMT_STATUS_SUCCESS)
return r;
if (!gpu_id && is_dgpu) {
if (!gpu_id && hsakmt_is_dgpu) {
/* Sharing non paged system memory. Use first GPU which was
* used during allocation. See fmm_allocate_host_gpu()
*/
@@ -3777,20 +3777,20 @@ HSAKMT_STATUS fmm_share_memory(void *MemoryAddress,
exportArgs.gpu_id = gpu_id;
exportArgs.flags = obj->mflags.Value;
r = kmtIoctl(kfd_fd, AMDKFD_IOC_IPC_EXPORT_HANDLE, (void *)&exportArgs);
r = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_IPC_EXPORT_HANDLE, (void *)&exportArgs);
if (r)
return HSAKMT_STATUS_ERROR;
memcpy(SharedMemoryStruct->ShareHandle, exportArgs.share_handle,
sizeof(SharedMemoryStruct->ShareHandle));
SharedMemoryStruct->ApeInfo = ApeInfo;
SharedMemoryStruct->SizeInPages = (HSAuint32) (SizeInBytes >> PAGE_SHIFT);
SharedMemoryStruct->SizeInPages = (HSAuint32) (SizeInBytes >> HSAKMT_PAGE_SHIFT);
SharedMemoryStruct->ExportGpuId = gpu_id;
return HSAKMT_STATUS_SUCCESS;
}
HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemoryHandle,
HSAKMT_STATUS hsakmt_fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemoryHandle,
HSAuint64 *SizeInBytes,
void **MemoryAddress,
uint32_t *gpu_id_array,
@@ -3821,7 +3821,7 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo
pthread_mutex_lock(&aperture->fmm_mutex);
reservedMem = aperture_allocate_area(aperture, NULL,
(SizeInPages << PAGE_SHIFT));
(SizeInPages << HSAKMT_PAGE_SHIFT));
pthread_mutex_unlock(&aperture->fmm_mutex);
if (!reservedMem) {
err = HSAKMT_STATUS_NO_MEMORY;
@@ -3829,7 +3829,7 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo
}
importArgs.va_addr = (uint64_t)reservedMem;
r = kmtIoctl(kfd_fd, AMDKFD_IOC_IPC_IMPORT_HANDLE, (void *)&importArgs);
r = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_IPC_IMPORT_HANDLE, (void *)&importArgs);
if (r) {
err = HSAKMT_STATUS_ERROR;
goto err_import;
@@ -3838,7 +3838,7 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo
pthread_mutex_lock(&aperture->fmm_mutex);
mflags.Value = importArgs.flags;
obj = aperture_allocate_object(aperture, reservedMem, importArgs.handle,
(SizeInPages << PAGE_SHIFT), mflags);
(SizeInPages << HSAKMT_PAGE_SHIFT), mflags);
if (!obj) {
err = HSAKMT_STATUS_NO_MEMORY;
goto err_free_mem;
@@ -3854,7 +3854,7 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo
goto err_free_obj;
}
obj->node_id = gpu_mem[gpu_mem_id].node_id;
ret = fmm_map_to_cpu(reservedMem, (SizeInPages << PAGE_SHIFT),
ret = fmm_map_to_cpu(reservedMem, (SizeInPages << HSAKMT_PAGE_SHIFT),
true, gpu_mem[gpu_mem_id].drm_render_fd,
importArgs.mmap_offset);
@@ -3865,7 +3865,7 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo
}
*MemoryAddress = reservedMem;
*SizeInBytes = (SizeInPages << PAGE_SHIFT);
*SizeInBytes = (SizeInPages << HSAKMT_PAGE_SHIFT);
if (gpu_id_array_size > 0) {
obj->registered_device_id_array = gpu_id_array;
@@ -3878,16 +3878,16 @@ err_free_obj:
pthread_mutex_lock(&aperture->fmm_mutex);
vm_remove_object(aperture, obj);
err_free_mem:
aperture_release_area(aperture, reservedMem, (SizeInPages << PAGE_SHIFT));
aperture_release_area(aperture, reservedMem, (SizeInPages << HSAKMT_PAGE_SHIFT));
pthread_mutex_unlock(&aperture->fmm_mutex);
err_free_buffer:
freeArgs.handle = importArgs.handle;
kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &freeArgs);
hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &freeArgs);
err_import:
return err;
}
HSAKMT_STATUS fmm_deregister_memory(void *address)
HSAKMT_STATUS hsakmt_fmm_deregister_memory(void *address)
{
manageable_aperture_t *aperture;
vm_object_t *object;
@@ -3897,7 +3897,7 @@ HSAKMT_STATUS fmm_deregister_memory(void *address)
/* On APUs we assume it's a random system memory address
* where registration and dergistration is a no-op
*/
return (!is_dgpu || is_svm_api_supported) ?
return (!hsakmt_is_dgpu || hsakmt_is_svm_api_supported) ?
HSAKMT_STATUS_SUCCESS :
HSAKMT_STATUS_MEMORY_NOT_REGISTERED;
/* Successful vm_find_object returns with aperture locked */
@@ -3946,7 +3946,7 @@ HSAKMT_STATUS fmm_deregister_memory(void *address)
* and maps nodes_to_map
*/
HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size,
HSAKMT_STATUS hsakmt_fmm_map_to_gpu_nodes(void *address, uint64_t size,
uint32_t *nodes_to_map, uint64_t num_of_nodes,
uint64_t *gpuvm_address)
{
@@ -3961,7 +3961,7 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size,
return HSAKMT_STATUS_INVALID_PARAMETER;
object = vm_find_object(address, size, &aperture);
if (!object && !is_svm_api_supported)
if (!object && !hsakmt_is_svm_api_supported)
return HSAKMT_STATUS_ERROR;
/* Successful vm_find_object returns with aperture locked */
@@ -3984,7 +3984,7 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size,
return HSAKMT_STATUS_ERROR;
}
if ((is_svm_api_supported && !object) || object->userptr) {
if ((hsakmt_is_svm_api_supported && !object) || object->userptr) {
retcode = _fmm_map_to_gpu_userptr(address, size, gpuvm_address,
object, nodes_to_map, num_of_nodes * sizeof(uint32_t));
if (object)
@@ -4061,7 +4061,7 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size,
return 0;
}
HSAKMT_STATUS fmm_get_mem_info(const void *address, HsaPointerInfo *info)
HSAKMT_STATUS hsakmt_fmm_get_mem_info(const void *address, HsaPointerInfo *info)
{
HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS;
uint32_t i;
@@ -4102,7 +4102,7 @@ HSAKMT_STATUS fmm_get_mem_info(const void *address, HsaPointerInfo *info)
* register to new nodes) or the memory being freed
*/
for (i = 0; i < info->NRegisteredNodes; i++)
gpuid_to_nodeid(vm_obj->registered_device_id_array[i],
hsakmt_gpuid_to_nodeid(vm_obj->registered_device_id_array[i],
&vm_obj->registered_node_id_array[i]);
}
info->RegisteredNodes = vm_obj->registered_node_id_array;
@@ -4117,7 +4117,7 @@ HSAKMT_STATUS fmm_get_mem_info(const void *address, HsaPointerInfo *info)
* to new nodes) or memory being freed
*/
for (i = 0; i < info->NMappedNodes; i++)
gpuid_to_nodeid(vm_obj->mapped_device_id_array[i],
hsakmt_gpuid_to_nodeid(vm_obj->mapped_device_id_array[i],
&vm_obj->mapped_node_id_array[i]);
}
info->MappedNodes = vm_obj->mapped_node_id_array;
@@ -4138,7 +4138,7 @@ HSAKMT_STATUS fmm_get_mem_info(const void *address, HsaPointerInfo *info)
}
#ifdef SANITIZER_AMDGPU
HSAKMT_STATUS fmm_replace_asan_header_page(void* address)
HSAKMT_STATUS hsakmt_fmm_replace_asan_header_page(void* address)
{
HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS;
manageable_aperture_t* aperture;
@@ -4165,7 +4165,7 @@ HSAKMT_STATUS fmm_replace_asan_header_page(void* address)
return ret;
}
HSAKMT_STATUS fmm_return_asan_header_page(void* address)
HSAKMT_STATUS hsakmt_fmm_return_asan_header_page(void* address)
{
HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS;
manageable_aperture_t* aperture;
@@ -4195,7 +4195,7 @@ HSAKMT_STATUS fmm_return_asan_header_page(void* address)
}
#endif
HSAKMT_STATUS fmm_set_mem_user_data(const void *mem, void *usr_data)
HSAKMT_STATUS hsakmt_fmm_set_mem_user_data(const void *mem, void *usr_data)
{
manageable_aperture_t *aperture;
vm_object_t *vm_obj;
@@ -4228,7 +4228,7 @@ static void fmm_clear_aperture(manageable_aperture_t *app)
* after a fork(). This will clear all vm_objects and mmaps duplicated from
* the parent.
*/
void fmm_clear_all_mem(void)
void hsakmt_fmm_clear_all_mem(void)
{
uint32_t i;
void *map_addr;
@@ -4252,7 +4252,7 @@ void fmm_clear_all_mem(void)
if (dgpu_shared_aperture_limit) {
/* Use the same dgpu range as the parent. If failed, then set
* is_dgpu_mem_init to false. Later on dgpu_mem_init will try
* hsakmt_is_dgpu_mem_init to false. Later on dgpu_mem_init will try
* to get a new range
*/
map_addr = mmap(dgpu_shared_aperture_base, (HSAuint64)(dgpu_shared_aperture_limit)-
@@ -4284,5 +4284,5 @@ void fmm_clear_all_mem(void)
fmm_clear_aperture(&gpu_mem[i].scratch_physical);
}
fmm_destroy_process_apertures();
hsakmt_fmm_destroy_process_apertures();
}
projects/rocr-runtime/libhsakmt/src/fmm.h
+27 -27
View File
@@ -45,63 +45,63 @@ typedef struct {
void *start_address;
} aperture_properties_t;
HSAKMT_STATUS fmm_get_amdgpu_device_handle(uint32_t node_id, HsaAMDGPUDeviceHandle *DeviceHandle);
HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes);
void fmm_destroy_process_apertures(void);
HSAKMT_STATUS hsakmt_fmm_get_amdgpu_device_handle(uint32_t node_id, HsaAMDGPUDeviceHandle *DeviceHandle);
HSAKMT_STATUS hsakmt_fmm_init_process_apertures(unsigned int NumNodes);
void hsakmt_fmm_destroy_process_apertures(void);
/* Memory interface */
void *fmm_allocate_scratch(uint32_t gpu_id, void *address, uint64_t MemorySizeInBytes);
void *fmm_allocate_device(uint32_t gpu_id, uint32_t node_id, void *address,
void *hsakmt_fmm_allocate_scratch(uint32_t gpu_id, void *address, uint64_t MemorySizeInBytes);
void *hsakmt_fmm_allocate_device(uint32_t gpu_id, uint32_t node_id, void *address,
uint64_t MemorySizeInBytes, uint64_t alignment, HsaMemFlags flags);
void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, uint64_t doorbell_offset);
void *fmm_allocate_host(uint32_t gpu_id, uint32_t node_id, void *address, uint64_t MemorySizeInBytes,
void *hsakmt_fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, uint64_t doorbell_offset);
void *hsakmt_fmm_allocate_host(uint32_t gpu_id, uint32_t node_id, void *address, uint64_t MemorySizeInBytes,
uint64_t alignment, HsaMemFlags flags);
void fmm_print(uint32_t node);
HSAKMT_STATUS fmm_release(void *address);
HSAKMT_STATUS fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address);
int fmm_unmap_from_gpu(void *address);
bool fmm_get_handle(void *address, uint64_t *handle);
HSAKMT_STATUS fmm_get_mem_info(const void *address, HsaPointerInfo *info);
HSAKMT_STATUS fmm_set_mem_user_data(const void *mem, void *usr_data);
void hsakmt_fmm_print(uint32_t node);
HSAKMT_STATUS hsakmt_fmm_release(void *address);
HSAKMT_STATUS hsakmt_fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address);
int hsakmt_fmm_unmap_from_gpu(void *address);
bool hsakmt_fmm_get_handle(void *address, uint64_t *handle);
HSAKMT_STATUS hsakmt_fmm_get_mem_info(const void *address, HsaPointerInfo *info);
HSAKMT_STATUS hsakmt_fmm_set_mem_user_data(const void *mem, void *usr_data);
#ifdef SANITIZER_AMDGPU
HSAKMT_STATUS fmm_replace_asan_header_page(void* address);
HSAKMT_STATUS fmm_return_asan_header_page(void* address);
HSAKMT_STATUS hsakmt_fmm_replace_asan_header_page(void* address);
HSAKMT_STATUS hsakmt_fmm_return_asan_header_page(void* address);
#endif
/* Topology interface*/
HSAKMT_STATUS fmm_node_added(HSAuint32 gpu_id);
HSAKMT_STATUS fmm_node_removed(HSAuint32 gpu_id);
HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSAuint32 gpu_id,
HSAKMT_STATUS hsakmt_fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSAuint32 gpu_id,
HSAuint64 *aperture_base, HSAuint64 *aperture_limit);
HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes,
HSAKMT_STATUS hsakmt_fmm_register_memory(void *address, uint64_t size_in_bytes,
uint32_t *gpu_id_array,
uint32_t gpu_id_array_size,
bool coarse_grain,
bool ext_coherent);
HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle,
HSAKMT_STATUS hsakmt_fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle,
HsaGraphicsResourceInfo *GraphicsResourceInfo,
uint32_t *gpu_id_array,
uint32_t gpu_id_array_size);
HSAKMT_STATUS fmm_deregister_memory(void *address);
HSAKMT_STATUS fmm_export_dma_buf_fd(void *MemoryAddress,
HSAKMT_STATUS hsakmt_fmm_deregister_memory(void *address);
HSAKMT_STATUS hsakmt_fmm_export_dma_buf_fd(void *MemoryAddress,
HSAuint64 MemorySizeInBytes,
int *DMABufFd,
HSAuint64 *Offset);
HSAKMT_STATUS fmm_share_memory(void *MemoryAddress,
HSAKMT_STATUS hsakmt_fmm_share_memory(void *MemoryAddress,
HSAuint64 SizeInBytes,
HsaSharedMemoryHandle *SharedMemoryHandle);
HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemoryHandle,
HSAKMT_STATUS hsakmt_fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemoryHandle,
HSAuint64 *SizeInBytes,
void **MemoryAddress,
uint32_t *gpu_id_array,
uint32_t gpu_id_array_size);
HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size,
HSAKMT_STATUS hsakmt_fmm_map_to_gpu_nodes(void *address, uint64_t size,
uint32_t *nodes_to_map, uint64_t num_of_nodes, uint64_t *gpuvm_address);
int open_drm_render_device(int minor);
void *mmap_allocate_aligned(int prot, int flags, uint64_t size, uint64_t align,
int hsakmt_open_drm_render_device(int minor);
void *hsakmt_mmap_allocate_aligned(int prot, int flags, uint64_t size, uint64_t align,
uint64_t guard_size, void *aper_base, void *aper_limit);
extern int (*fn_amdgpu_device_get_fd)(HsaAMDGPUDeviceHandle device_handle);
extern int (*hsakmt_fn_amdgpu_device_get_fd)(HsaAMDGPUDeviceHandle device_handle);
#endif /* FMM_H_ */
projects/rocr-runtime/libhsakmt/src/globals.c
+7 -7
View File
@@ -27,19 +27,19 @@
// HSAKMT global data
int kfd_fd = -1;
unsigned long kfd_open_count;
unsigned long system_properties_count;
int hsakmt_kfd_fd = -1;
unsigned long hsakmt_kfd_open_count;
unsigned long hsakmt_system_properties_count;
pthread_mutex_t hsakmt_mutex = PTHREAD_MUTEX_INITIALIZER;
bool is_dgpu;
bool hsakmt_is_dgpu;
#ifndef PAGE_SIZE
int PAGE_SIZE;
#endif
int PAGE_SHIFT;
int HSAKMT_PAGE_SHIFT;
/* whether to check all dGPUs in the topology support SVM API */
bool is_svm_api_supported;
bool hsakmt_is_svm_api_supported;
/* zfb is mainly used during emulation */
int zfb_support;
int hsakmt_zfb_support;
projects/rocr-runtime/libhsakmt/src/libhsakmt.c
+2 -2
View File
@@ -5,7 +5,7 @@
#include "libhsakmt.h"
/* Call ioctl, restarting if it is interrupted */
int kmtIoctl(int fd, unsigned long request, void *arg)
int hsakmt_ioctl(int fd, unsigned long request, void *arg)
{
int ret;
@@ -18,7 +18,7 @@ int kmtIoctl(int fd, unsigned long request, void *arg)
* make any subsequent hsaKmt calls fail in CHECK_KFD_OPEN.
*/
pr_err("KFD file descriptor not valid in this process\n");
is_forked_child();
hsakmt_is_forked_child();
}
return ret;
projects/rocr-runtime/libhsakmt/src/libhsakmt.h
+41 -41
View File
@@ -32,15 +32,15 @@
#include <stdint.h>
#include <limits.h>
extern int kfd_fd;
extern unsigned long kfd_open_count;
extern int hsakmt_kfd_fd;
extern unsigned long hsakmt_kfd_open_count;
extern bool hsakmt_forked;
extern pthread_mutex_t hsakmt_mutex;
extern bool is_dgpu;
extern bool is_svm_api_supported;
extern int zfb_support;
extern bool hsakmt_is_dgpu;
extern bool hsakmt_is_svm_api_supported;
extern int hsakmt_zfb_support;
extern HsaVersionInfo kfd_version_info;
extern HsaVersionInfo hsakmt_kfd_version_info;
#undef HSAKMTAPI
#define HSAKMTAPI __attribute__((visibility ("default")))
@@ -58,10 +58,10 @@ extern HsaVersionInfo kfd_version_info;
#define PORT_UINT64_TO_VPTR(v) ((void*)(unsigned long)(v))
#define CHECK_KFD_OPEN() \
do { if (kfd_open_count == 0 || hsakmt_forked) return HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED; } while (0)
do { if (hsakmt_kfd_open_count == 0 || hsakmt_forked) return HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED; } while (0)
#define CHECK_KFD_MINOR_VERSION(minor) \
do { if ((minor) > kfd_version_info.KernelInterfaceMinorVersion)\
do { if ((minor) > hsakmt_kfd_version_info.KernelInterfaceMinorVersion)\
return HSAKMT_STATUS_NOT_SUPPORTED; } while (0)
/* Might be defined in limits.h on platforms where it is constant (used by musl) */
@@ -69,7 +69,7 @@ extern HsaVersionInfo kfd_version_info;
#ifndef PAGE_SIZE
extern int PAGE_SIZE;
#endif
extern int PAGE_SHIFT;
extern int HSAKMT_PAGE_SHIFT;
/* VI HW bug requires this virtual address alignment */
#define TONGA_PAGE_SIZE 0x8000
@@ -174,47 +174,47 @@ struct hsa_gfxip_table {
const char *amd_name; // CALName of the device
};
HSAKMT_STATUS init_kfd_version(void);
HSAKMT_STATUS hsakmt_init_kfd_version(void);
#define IS_SOC15(gfxv) ((gfxv) >= GFX_VERSION_VEGA10)
HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id);
HSAKMT_STATUS gpuid_to_nodeid(uint32_t gpu_id, uint32_t* node_id);
uint32_t get_gfxv_by_node_id(HSAuint32 node_id);
bool prefer_ats(HSAuint32 node_id);
uint16_t get_device_id_by_node_id(HSAuint32 node_id);
uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id);
uint32_t get_direct_link_cpu(uint32_t gpu_node);
HSAKMT_STATUS hsakmt_validate_nodeid(uint32_t nodeid, uint32_t *gpu_id);
HSAKMT_STATUS hsakmt_gpuid_to_nodeid(uint32_t gpu_id, uint32_t* node_id);
uint32_t hsakmt_get_gfxv_by_node_id(HSAuint32 node_id);
bool hsakmt_prefer_ats(HSAuint32 node_id);
uint16_t hsakmt_get_device_id_by_node_id(HSAuint32 node_id);
uint16_t hsakmt_get_device_id_by_gpu_id(HSAuint32 gpu_id);
uint32_t hsakmt_get_direct_link_cpu(uint32_t gpu_node);
int get_drm_render_fd_by_gpu_id(HSAuint32 gpu_id);
HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array,
HSAKMT_STATUS hsakmt_validate_nodeid_array(uint32_t **gpu_id_array,
uint32_t NumberOfNodes, uint32_t *NodeArray);
HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props);
HSAKMT_STATUS topology_get_node_props(HSAuint32 NodeId,
HSAKMT_STATUS hsakmt_topology_sysfs_get_system_props(HsaSystemProperties *props);
HSAKMT_STATUS hsakmt_topology_get_node_props(HSAuint32 NodeId,
HsaNodeProperties *NodeProperties);
HSAKMT_STATUS topology_get_iolink_props(HSAuint32 NodeId,
HSAKMT_STATUS hsakmt_topology_get_iolink_props(HSAuint32 NodeId,
HSAuint32 NumIoLinks,
HsaIoLinkProperties *IoLinkProperties);
void topology_setup_is_dgpu_param(HsaNodeProperties *props);
bool topology_is_svm_needed(HSA_ENGINE_ID EngineId);
void hsakmt_topology_setup_is_dgpu_param(HsaNodeProperties *props);
bool hsakmt_topology_is_svm_needed(HSA_ENGINE_ID EngineId);
HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags);
HSAuint32 hsakmt_PageSizeFromFlags(unsigned int pageSizeFlags);
void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align,
void* hsakmt_allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align,
uint32_t gpu_id,
uint32_t NodeId, bool NonPaged,
bool DeviceLocal, bool Uncached);
void free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align);
HSAKMT_STATUS init_process_doorbells(unsigned int NumNodes);
void destroy_process_doorbells(void);
HSAKMT_STATUS init_device_debugging_memory(unsigned int NumNodes);
void destroy_device_debugging_memory(void);
bool debug_get_reg_status(uint32_t node_id);
HSAKMT_STATUS init_counter_props(unsigned int NumNodes);
void destroy_counter_props(void);
uint32_t *convert_queue_ids(HSAuint32 NumQueues, HSA_QUEUEID *Queues);
void hsakmt_free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align);
HSAKMT_STATUS hsakmt_init_process_doorbells(unsigned int NumNodes);
void hsakmt_destroy_process_doorbells(void);
HSAKMT_STATUS hsakmt_init_device_debugging_memory(unsigned int NumNodes);
void hsakmt_destroy_device_debugging_memory(void);
bool hsakmt_debug_get_reg_status(uint32_t node_id);
HSAKMT_STATUS hsakmt_init_counter_props(unsigned int NumNodes);
void hsakmt_destroy_counter_props(void);
uint32_t *hsakmt_convert_queue_ids(HSAuint32 NumQueues, HSA_QUEUEID *Queues);
extern int kmtIoctl(int fd, unsigned long request, void *arg);
extern int hsakmt_ioctl(int fd, unsigned long request, void *arg);
/* Void pointer arithmetic (or remove -Wpointer-arith to allow void pointers arithmetic) */
#define VOID_PTR_ADD32(ptr,n) (void*)((uint32_t*)(ptr) + n)/*ptr + offset*/
@@ -230,14 +230,14 @@ extern int kmtIoctl(int fd, unsigned long request, void *arg);
typeof(a) tmp1 = (a), tmp2 = (b); \
tmp1 > tmp2 ? tmp1 : tmp2; })
void clear_events_page(void);
void fmm_clear_all_mem(void);
void clear_process_doorbells(void);
uint32_t get_num_sysfs_nodes(void);
void hsakmt_clear_events_page(void);
void hsakmt_fmm_clear_all_mem(void);
void hsakmt_clear_process_doorbells(void);
uint32_t hsakmt_get_num_sysfs_nodes(void);
bool is_forked_child(void);
bool hsakmt_is_forked_child(void);
/* Calculate VGPR and SGPR register file size per CU */
uint32_t get_vgpr_size_per_cu(uint32_t gfxv);
uint32_t hsakmt_get_vgpr_size_per_cu(uint32_t gfxv);
#define SGPR_SIZE_PER_CU 0x4000
#endif
projects/rocr-runtime/libhsakmt/src/memory.c
+40 -40
View File
@@ -49,11 +49,11 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetMemoryPolicy(HSAuint32 Node,
pr_debug("[%s] node %d; default %d; alternate %d\n",
__func__, Node, DefaultPolicy, AlternatePolicy);
result = validate_nodeid(Node, &gpu_id);
result = hsakmt_validate_nodeid(Node, &gpu_id);
if (result != HSAKMT_STATUS_SUCCESS)
return result;
if (get_gfxv_by_node_id(Node) != GFX_VERSION_KAVERI)
if (hsakmt_get_gfxv_by_node_id(Node) != GFX_VERSION_KAVERI)
/* This is a legacy API useful on Kaveri only. On dGPU
* the alternate aperture is setup and used
* automatically for coherent allocations. Don't let
@@ -86,12 +86,12 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetMemoryPolicy(HSAuint32 Node,
args.alternate_aperture_base = (uintptr_t) MemoryAddressAlternate;
args.alternate_aperture_size = MemorySizeInBytes;
int err = kmtIoctl(kfd_fd, AMDKFD_IOC_SET_MEMORY_POLICY, &args);
int err = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_SET_MEMORY_POLICY, &args);
return (err == -1) ? HSAKMT_STATUS_ERROR : HSAKMT_STATUS_SUCCESS;
}
HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags)
HSAuint32 hsakmt_PageSizeFromFlags(unsigned int pageSizeFlags)
{
switch (pageSizeFlags) {
case HSA_PAGE_SIZE_4KB: return 4*1024;
@@ -131,13 +131,13 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemoryAlign(HSAuint32 PreferredNode,
pr_debug("[%s] node %d\n", __func__, PreferredNode);
result = validate_nodeid(PreferredNode, &gpu_id);
result = hsakmt_validate_nodeid(PreferredNode, &gpu_id);
if (result != HSAKMT_STATUS_SUCCESS) {
pr_err("[%s] invalid node ID: %d\n", __func__, PreferredNode);
return result;
}
page_size = PageSizeFromFlags(MemFlags.ui32.PageSize);
page_size = hsakmt_PageSizeFromFlags(MemFlags.ui32.PageSize);
if (Alignment && (Alignment < page_size || !POWER_OF_2(Alignment)))
return HSAKMT_STATUS_INVALID_PARAMETER;
@@ -163,7 +163,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemoryAlign(HSAuint32 PreferredNode,
return HSAKMT_STATUS_NOT_IMPLEMENTED;
}
*MemoryAddress = fmm_allocate_scratch(gpu_id, *MemoryAddress, SizeInBytes);
*MemoryAddress = hsakmt_fmm_allocate_scratch(gpu_id, *MemoryAddress, SizeInBytes);
if (!(*MemoryAddress)) {
pr_err("[%s] failed to allocate %lu bytes from scratch\n",
@@ -175,16 +175,16 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemoryAlign(HSAuint32 PreferredNode,
}
/* GPU allocated system memory */
if (!gpu_id || !MemFlags.ui32.NonPaged || zfb_support || MemFlags.ui32.GTTAccess) {
if (!gpu_id || !MemFlags.ui32.NonPaged || hsakmt_zfb_support || MemFlags.ui32.GTTAccess) {
/* Backwards compatibility hack: Allocate system memory if app
* asks for paged memory from a GPU node.
*/
/* If allocate VRAM under ZFB mode */
if (zfb_support && gpu_id && MemFlags.ui32.NonPaged == 1)
if (hsakmt_zfb_support && gpu_id && MemFlags.ui32.NonPaged == 1)
MemFlags.ui32.CoarseGrain = 1;
*MemoryAddress = fmm_allocate_host(gpu_id, MemFlags.ui32.GTTAccess ? 0 : PreferredNode,
*MemoryAddress = hsakmt_fmm_allocate_host(gpu_id, MemFlags.ui32.GTTAccess ? 0 : PreferredNode,
*MemoryAddress, SizeInBytes, Alignment, MemFlags);
if (!(*MemoryAddress)) {
@@ -204,7 +204,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemoryAlign(HSAuint32 PreferredNode,
return HSAKMT_STATUS_INVALID_PARAMETER;
}
*MemoryAddress = fmm_allocate_device(gpu_id, PreferredNode, *MemoryAddress,
*MemoryAddress = hsakmt_fmm_allocate_device(gpu_id, PreferredNode, *MemoryAddress,
SizeInBytes, Alignment, MemFlags);
if (!(*MemoryAddress)) {
@@ -229,7 +229,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtFreeMemory(void *MemoryAddress,
return HSAKMT_STATUS_ERROR;
}
return fmm_release(MemoryAddress);
return hsakmt_fmm_release(MemoryAddress);
}
HSAKMT_STATUS HSAKMTAPI hsaKmtAvailableMemory(HSAuint32 Node,
@@ -243,13 +243,13 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAvailableMemory(HSAuint32 Node,
pr_debug("[%s] node %d\n", __func__, Node);
result = validate_nodeid(Node, &args.gpu_id);
result = hsakmt_validate_nodeid(Node, &args.gpu_id);
if (result != HSAKMT_STATUS_SUCCESS) {
pr_err("[%s] invalid node ID: %d\n", __func__, Node);
return result;
}
if (kmtIoctl(kfd_fd, AMDKFD_IOC_AVAILABLE_MEMORY, &args))
if (hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_AVAILABLE_MEMORY, &args))
return HSAKMT_STATUS_ERROR;
*AvailableBytes = args.available;
@@ -263,11 +263,11 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemory(void *MemoryAddress,
pr_debug("[%s] address %p\n", __func__, MemoryAddress);
if (!is_dgpu)
if (!hsakmt_is_dgpu)
/* TODO: support mixed APU and dGPU configurations */
return HSAKMT_STATUS_SUCCESS;
return fmm_register_memory(MemoryAddress, MemorySizeInBytes,
return hsakmt_fmm_register_memory(MemoryAddress, MemorySizeInBytes,
NULL, 0, true, false);
}
@@ -283,15 +283,15 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemoryToNodes(void *MemoryAddress,
pr_debug("[%s] address %p number of nodes %lu\n",
__func__, MemoryAddress, NumberOfNodes);
if (!is_dgpu)
if (!hsakmt_is_dgpu)
/* TODO: support mixed APU and dGPU configurations */
return HSAKMT_STATUS_NOT_SUPPORTED;
ret = validate_nodeid_array(&gpu_id_array,
ret = hsakmt_validate_nodeid_array(&gpu_id_array,
NumberOfNodes, NodeArray);
if (ret == HSAKMT_STATUS_SUCCESS) {
ret = fmm_register_memory(MemoryAddress, MemorySizeInBytes,
ret = hsakmt_fmm_register_memory(MemoryAddress, MemorySizeInBytes,
gpu_id_array,
NumberOfNodes*sizeof(uint32_t),
true, false);
@@ -319,11 +319,11 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemoryWithFlags(void *MemoryAddress,
if ((MemFlags.ui32.HostAccess != 1) || (MemFlags.ui32.NonPaged == 1))
return HSAKMT_STATUS_NOT_SUPPORTED;
if (!is_dgpu)
if (!hsakmt_is_dgpu)
/* TODO: support mixed APU and dGPU configurations */
return HSAKMT_STATUS_NOT_SUPPORTED;
ret = fmm_register_memory(MemoryAddress, MemorySizeInBytes,
ret = hsakmt_fmm_register_memory(MemoryAddress, MemorySizeInBytes,
NULL, 0, MemFlags.ui32.CoarseGrain, MemFlags.ui32.ExtendedCoherent);
return ret;
@@ -341,12 +341,12 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterGraphicsHandleToNodes(HSAuint64 GraphicsRe
pr_debug("[%s] number of nodes %lu\n", __func__, NumberOfNodes);
if (NodeArray != NULL || NumberOfNodes != 0) {
ret = validate_nodeid_array(&gpu_id_array,
ret = hsakmt_validate_nodeid_array(&gpu_id_array,
NumberOfNodes, NodeArray);
}
if (ret == HSAKMT_STATUS_SUCCESS) {
ret = fmm_register_graphics_handle(
ret = hsakmt_fmm_register_graphics_handle(
GraphicsResourceHandle, GraphicsResourceInfo,
gpu_id_array, NumberOfNodes * sizeof(uint32_t));
if (ret != HSAKMT_STATUS_SUCCESS)
@@ -366,7 +366,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtExportDMABufHandle(void *MemoryAddress,
pr_debug("[%s] address %p\n", __func__, MemoryAddress);
return fmm_export_dma_buf_fd(MemoryAddress, MemorySizeInBytes,
return hsakmt_fmm_export_dma_buf_fd(MemoryAddress, MemorySizeInBytes,
DMABufFd, Offset);
}
@@ -381,7 +381,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtShareMemory(void *MemoryAddress,
if (!SharedMemoryHandle)
return HSAKMT_STATUS_INVALID_PARAMETER;
return fmm_share_memory(MemoryAddress, SizeInBytes, SharedMemoryHandle);
return hsakmt_fmm_share_memory(MemoryAddress, SizeInBytes, SharedMemoryHandle);
}
HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterSharedHandle(const HsaSharedMemoryHandle *SharedMemoryHandle,
@@ -417,12 +417,12 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterSharedHandleToNodes(const HsaSharedMemoryH
return HSAKMT_STATUS_INVALID_PARAMETER;
if (NodeArray) {
ret = validate_nodeid_array(&gpu_id_array, NumberOfNodes, NodeArray);
ret = hsakmt_validate_nodeid_array(&gpu_id_array, NumberOfNodes, NodeArray);
if (ret != HSAKMT_STATUS_SUCCESS)
goto error;
}
ret = fmm_register_shared_memory(SharedMemoryHandle,
ret = hsakmt_fmm_register_shared_memory(SharedMemoryHandle,
SizeInBytes,
MemoryAddress,
gpu_id_array,
@@ -469,7 +469,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDeregisterMemory(void *MemoryAddress)
pr_debug("[%s] address %p\n", __func__, MemoryAddress);
return fmm_deregister_memory(MemoryAddress);
return hsakmt_fmm_deregister_memory(MemoryAddress);
}
HSAKMT_STATUS HSAKMTAPI hsaKmtMapMemoryToGPU(void *MemoryAddress,
@@ -488,7 +488,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtMapMemoryToGPU(void *MemoryAddress,
if (AlternateVAGPU)
*AlternateVAGPU = 0;
return fmm_map_to_gpu(MemoryAddress, MemorySizeInBytes, AlternateVAGPU);
return hsakmt_fmm_map_to_gpu(MemoryAddress, MemorySizeInBytes, AlternateVAGPU);
}
HSAKMT_STATUS HSAKMTAPI hsaKmtMapMemoryToGPUNodes(void *MemoryAddress,
@@ -511,17 +511,17 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtMapMemoryToGPUNodes(void *MemoryAddress,
return HSAKMT_STATUS_ERROR;
}
if (!is_dgpu && NumberOfNodes == 1)
if (!hsakmt_is_dgpu && NumberOfNodes == 1)
return hsaKmtMapMemoryToGPU(MemoryAddress,
MemorySizeInBytes,
AlternateVAGPU);
ret = validate_nodeid_array(&gpu_id_array,
ret = hsakmt_validate_nodeid_array(&gpu_id_array,
NumberOfNodes, NodeArray);
if (ret != HSAKMT_STATUS_SUCCESS)
return ret;
ret = fmm_map_to_gpu_nodes(MemoryAddress, MemorySizeInBytes,
ret = hsakmt_fmm_map_to_gpu_nodes(MemoryAddress, MemorySizeInBytes,
gpu_id_array, NumberOfNodes, AlternateVAGPU);
if (gpu_id_array)
@@ -542,7 +542,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtUnmapMemoryToGPU(void *MemoryAddress)
return HSAKMT_STATUS_SUCCESS;
}
if (!fmm_unmap_from_gpu(MemoryAddress))
if (!hsakmt_fmm_unmap_from_gpu(MemoryAddress))
return HSAKMT_STATUS_SUCCESS;
else
return HSAKMT_STATUS_ERROR;
@@ -582,7 +582,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetTileConfig(HSAuint32 NodeId, HsaGpuTileConfig *
pr_debug("[%s] node %d\n", __func__, NodeId);
result = validate_nodeid(NodeId, &gpu_id);
result = hsakmt_validate_nodeid(NodeId, &gpu_id);
if (result != HSAKMT_STATUS_SUCCESS)
return result;
@@ -598,7 +598,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetTileConfig(HSAuint32 NodeId, HsaGpuTileConfig *
args.num_tile_configs = config->NumTileConfigs;
args.num_macro_tile_configs = config->NumMacroTileConfigs;
if (kmtIoctl(kfd_fd, AMDKFD_IOC_GET_TILE_CONFIG, &args) != 0)
if (hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_GET_TILE_CONFIG, &args) != 0)
return HSAKMT_STATUS_ERROR;
config->NumTileConfigs = args.num_tile_configs;
@@ -621,7 +621,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtQueryPointerInfo(const void *Pointer,
if (!PointerInfo)
return HSAKMT_STATUS_INVALID_PARAMETER;
return fmm_get_mem_info(Pointer, PointerInfo);
return hsakmt_fmm_get_mem_info(Pointer, PointerInfo);
}
HSAKMT_STATUS HSAKMTAPI hsaKmtSetMemoryUserData(const void *Pointer,
@@ -631,7 +631,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetMemoryUserData(const void *Pointer,
pr_debug("[%s] pointer %p\n", __func__, Pointer);
return fmm_set_mem_user_data(Pointer, UserData);
return hsakmt_fmm_set_mem_user_data(Pointer, UserData);
}
HSAKMT_STATUS HSAKMTAPI hsaKmtReplaceAsanHeaderPage(void *addr)
@@ -640,7 +640,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtReplaceAsanHeaderPage(void *addr)
pr_debug("[%s] address %p\n", __func__, addr);
CHECK_KFD_OPEN();
return fmm_replace_asan_header_page(addr);
return hsakmt_fmm_replace_asan_header_page(addr);
#else
return HSAKMT_STATUS_NOT_SUPPORTED;
#endif
@@ -652,7 +652,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtReturnAsanHeaderPage(void *addr)
pr_debug("[%s] address %p\n", __func__, addr);
CHECK_KFD_OPEN();
return fmm_return_asan_header_page(addr);
return hsakmt_fmm_return_asan_header_page(addr);
#else
return HSAKMT_STATUS_NOT_SUPPORTED;
#endif
@@ -663,5 +663,5 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetAMDGPUDeviceHandle( HSAuint32 NodeId,
{
CHECK_KFD_OPEN();
return fmm_get_amdgpu_device_handle(NodeId, DeviceHandle);
return hsakmt_fmm_get_amdgpu_device_handle(NodeId, DeviceHandle);
}
projects/rocr-runtime/libhsakmt/src/openclose.c
+31 -31
View File
@@ -42,19 +42,19 @@
#include <dlfcn.h>
#include <string.h>
int (*fn_amdgpu_device_get_fd)(HsaAMDGPUDeviceHandle device_handle);
int (*hsakmt_fn_amdgpu_device_get_fd)(HsaAMDGPUDeviceHandle device_handle);
static const char kfd_device_name[] = "/dev/kfd";
static pid_t parent_pid = -1;
int hsakmt_debug_level;
bool hsakmt_forked;
/* is_forked_child detects when the process has forked since the last
/* hsakmt_is_forked_child detects when the process has forked since the last
* time this function was called. We cannot rely on pthread_atfork
* because the process can fork without calling the fork function in
* libc (using clone or calling the system call directly).
*/
bool is_forked_child(void)
bool hsakmt_is_forked_child(void)
{
pid_t cur_pid;
@@ -99,15 +99,15 @@ static void child_fork_handler(void)
*/
static void clear_after_fork(void)
{
clear_process_doorbells();
clear_events_page();
fmm_clear_all_mem();
destroy_device_debugging_memory();
if (kfd_fd) {
close(kfd_fd);
kfd_fd = -1;
hsakmt_clear_process_doorbells();
hsakmt_clear_events_page();
hsakmt_fmm_clear_all_mem();
hsakmt_destroy_device_debugging_memory();
if (hsakmt_kfd_fd) {
close(hsakmt_kfd_fd);
hsakmt_kfd_fd = -1;
}
kfd_open_count = 0;
hsakmt_kfd_open_count = 0;
parent_pid = -1;
hsakmt_forked = false;
}
@@ -117,7 +117,7 @@ static inline void init_page_size(void)
#ifndef PAGE_SIZE
PAGE_SIZE = sysconf(_SC_PAGESIZE);
#endif
PAGE_SHIFT = ffs(PAGE_SIZE) - 1;
HSAKMT_PAGE_SHIFT = ffs(PAGE_SIZE) - 1;
}
static HSAKMT_STATUS init_vars_from_env(void)
@@ -141,7 +141,7 @@ static HSAKMT_STATUS init_vars_from_env(void)
/* Check whether to support Zero frame buffer */
envvar = getenv("HSA_ZFB");
if (envvar)
zfb_support = atoi(envvar);
hsakmt_zfb_support = atoi(envvar);
return HSAKMT_STATUS_SUCCESS;
}
@@ -157,26 +157,26 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void)
pthread_mutex_lock(&hsakmt_mutex);
/* If the process has forked, the child process must re-initialize
* it's connection to KFD. Any references tracked by kfd_open_count
* it's connection to KFD. Any references tracked by hsakmt_kfd_open_count
* belong to the parent
*/
if (is_forked_child())
if (hsakmt_is_forked_child())
clear_after_fork();
if (kfd_open_count == 0) {
if (hsakmt_kfd_open_count == 0) {
static bool atfork_installed = false;
fn_amdgpu_device_get_fd = dlsym(RTLD_DEFAULT, "amdgpu_device_get_fd");
hsakmt_fn_amdgpu_device_get_fd = dlsym(RTLD_DEFAULT, "amdgpu_device_get_fd");
if ((error = dlerror()) != NULL)
pr_err("amdgpu_device_get_fd is not available: %s\n", error);
else
pr_info("amdgpu_device_get_fd is available %p\n", fn_amdgpu_device_get_fd);
pr_info("amdgpu_device_get_fd is available %p\n", hsakmt_fn_amdgpu_device_get_fd);
result = init_vars_from_env();
if (result != HSAKMT_STATUS_SUCCESS)
goto open_failed;
if (kfd_fd < 0) {
if (hsakmt_kfd_fd < 0) {
fd = open(kfd_device_name, O_RDWR | O_CLOEXEC);
if (fd == -1) {
@@ -184,28 +184,28 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void)
goto open_failed;
}
kfd_fd = fd;
hsakmt_kfd_fd = fd;
}
init_page_size();
result = init_kfd_version();
result = hsakmt_init_kfd_version();
if (result != HSAKMT_STATUS_SUCCESS)
goto kfd_version_failed;
useSvmStr = getenv("HSA_USE_SVM");
is_svm_api_supported = !(useSvmStr && !strcmp(useSvmStr, "0"));
hsakmt_is_svm_api_supported = !(useSvmStr && !strcmp(useSvmStr, "0"));
result = topology_sysfs_get_system_props(&sys_props);
result = hsakmt_topology_sysfs_get_system_props(&sys_props);
if (result != HSAKMT_STATUS_SUCCESS)
goto topology_sysfs_failed;
kfd_open_count = 1;
hsakmt_kfd_open_count = 1;
if (init_device_debugging_memory(sys_props.NumNodes) != HSAKMT_STATUS_SUCCESS)
if (hsakmt_init_device_debugging_memory(sys_props.NumNodes) != HSAKMT_STATUS_SUCCESS)
pr_warn("Insufficient Memory. Debugging unavailable\n");
init_counter_props(sys_props.NumNodes);
hsakmt_init_counter_props(sys_props.NumNodes);
if (!atfork_installed) {
/* Atfork handlers cannot be uninstalled and
@@ -219,7 +219,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void)
atfork_installed = true;
}
} else {
kfd_open_count++;
hsakmt_kfd_open_count++;
result = HSAKMT_STATUS_KERNEL_ALREADY_OPENED;
}
@@ -240,10 +240,10 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCloseKFD(void)
pthread_mutex_lock(&hsakmt_mutex);
if (kfd_open_count > 0) {
if (--kfd_open_count == 0) {
destroy_counter_props();
destroy_device_debugging_memory();
if (hsakmt_kfd_open_count > 0) {
if (--hsakmt_kfd_open_count == 0) {
hsakmt_destroy_counter_props();
hsakmt_destroy_device_debugging_memory();
}
result = HSAKMT_STATUS_SUCCESS;
projects/rocr-runtime/libhsakmt/src/pc_sampling.c
+10 -10
View File
@@ -52,7 +52,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPcSamplingQueryCapabilities(HSAuint32 NodeId, void
CHECK_KFD_OPEN();
CHECK_KFD_MINOR_VERSION(16);
HSAKMT_STATUS ret = validate_nodeid(NodeId, &gpu_id);
HSAKMT_STATUS ret = hsakmt_validate_nodeid(NodeId, &gpu_id);
if (ret != HSAKMT_STATUS_SUCCESS) {
pr_err("[%s] invalid node ID: %d\n", __func__, NodeId);
return ret;
@@ -65,7 +65,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPcSamplingQueryCapabilities(HSAuint32 NodeId, void
args.num_sample_info = sample_info_sz;
args.flags = 0;
int err = kmtIoctl(kfd_fd, AMDKFD_IOC_PC_SAMPLE, &args);
int err = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_PC_SAMPLE, &args);
*size = args.num_sample_info;
@@ -99,7 +99,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPcSamplingCreate(HSAuint32 NodeId, HsaPcSamplingIn
CHECK_KFD_OPEN();
*traceId = INVALID_TRACE_ID;
HSAKMT_STATUS ret = validate_nodeid(NodeId, &gpu_id);
HSAKMT_STATUS ret = hsakmt_validate_nodeid(NodeId, &gpu_id);
if (ret != HSAKMT_STATUS_SUCCESS) {
pr_err("[%s] invalid node ID: %d\n", __func__, NodeId);
return ret;
@@ -111,7 +111,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPcSamplingCreate(HSAuint32 NodeId, HsaPcSamplingIn
args.num_sample_info = 1;
args.trace_id = INVALID_TRACE_ID;
int err = kmtIoctl(kfd_fd, AMDKFD_IOC_PC_SAMPLE, &args);
int err = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_PC_SAMPLE, &args);
if (err) {
switch (errno) {
case EINVAL:
@@ -139,7 +139,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPcSamplingDestroy(HSAuint32 NodeId, HsaPcSamplingT
CHECK_KFD_OPEN();
HSAKMT_STATUS ret = validate_nodeid(NodeId, &gpu_id);
HSAKMT_STATUS ret = hsakmt_validate_nodeid(NodeId, &gpu_id);
if (ret != HSAKMT_STATUS_SUCCESS) {
pr_err("[%s] invalid node ID: %d\n", __func__, NodeId);
return ret;
@@ -151,7 +151,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPcSamplingDestroy(HSAuint32 NodeId, HsaPcSamplingT
args.gpu_id = gpu_id;
args.trace_id = traceId;
int err = kmtIoctl(kfd_fd, AMDKFD_IOC_PC_SAMPLE, &args);
int err = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_PC_SAMPLE, &args);
if (err) {
if (errno == EINVAL)
return HSAKMT_STATUS_INVALID_PARAMETER;
@@ -171,7 +171,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPcSamplingStart(HSAuint32 NodeId, HsaPcSamplingTra
CHECK_KFD_OPEN();
HSAKMT_STATUS ret = validate_nodeid(NodeId, &gpu_id);
HSAKMT_STATUS ret = hsakmt_validate_nodeid(NodeId, &gpu_id);
if (ret != HSAKMT_STATUS_SUCCESS) {
pr_err("[%s] invalid node ID: %d\n", __func__, NodeId);
return ret;
@@ -181,7 +181,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPcSamplingStart(HSAuint32 NodeId, HsaPcSamplingTra
args.gpu_id = gpu_id;
args.trace_id = traceId;
int err = kmtIoctl(kfd_fd, AMDKFD_IOC_PC_SAMPLE, &args);
int err = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_PC_SAMPLE, &args);
if (err) {
switch (errno) {
case EINVAL:
@@ -210,7 +210,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPcSamplingStop(HSAuint32 NodeId, HsaPcSamplingTrac
CHECK_KFD_OPEN();
HSAKMT_STATUS ret = validate_nodeid(NodeId, &gpu_id);
HSAKMT_STATUS ret = hsakmt_validate_nodeid(NodeId, &gpu_id);
if (ret != HSAKMT_STATUS_SUCCESS) {
pr_err("[%s] invalid node ID: %d\n", __func__, NodeId);
return ret;
@@ -220,7 +220,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPcSamplingStop(HSAuint32 NodeId, HsaPcSamplingTrac
args.gpu_id = gpu_id;
args.trace_id = traceId;
int err = kmtIoctl(kfd_fd, AMDKFD_IOC_PC_SAMPLE, &args);
int err = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_PC_SAMPLE, &args);
if (err) {
switch (errno) {
case EINVAL:
projects/rocr-runtime/libhsakmt/src/perfctr.c
+8 -8
View File
@@ -99,7 +99,7 @@ static ssize_t readn(int fd, void *buf, size_t n)
return n;
}
HSAKMT_STATUS init_counter_props(unsigned int NumNodes)
HSAKMT_STATUS hsakmt_init_counter_props(unsigned int NumNodes)
{
counter_props = calloc(NumNodes, sizeof(struct HsaCounterProperties *));
if (!counter_props) {
@@ -112,7 +112,7 @@ HSAKMT_STATUS init_counter_props(unsigned int NumNodes)
return HSAKMT_STATUS_SUCCESS;
}
void destroy_counter_props(void)
void hsakmt_destroy_counter_props(void)
{
unsigned int i;
@@ -272,7 +272,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPmcGetCounterProperties(HSAuint32 NodeId,
if (!CounterProperties)
return HSAKMT_STATUS_INVALID_PARAMETER;
if (validate_nodeid(NodeId, &gpu_id) != HSAKMT_STATUS_SUCCESS)
if (hsakmt_validate_nodeid(NodeId, &gpu_id) != HSAKMT_STATUS_SUCCESS)
return HSAKMT_STATUS_INVALID_NODE_UNIT;
if (counter_props[NodeId]) {
@@ -281,7 +281,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPmcGetCounterProperties(HSAuint32 NodeId,
}
for (i = 0; i < PERFCOUNTER_BLOCKID__MAX; i++) {
rc = get_block_properties(NodeId, i, &block);
rc = hsakmt_get_block_properties(NodeId, i, &block);
if (rc != HSAKMT_STATUS_SUCCESS)
return rc;
total_concurrent += block.num_of_slots;
@@ -304,7 +304,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPmcGetCounterProperties(HSAuint32 NodeId,
block_prop = &counter_props[NodeId]->Blocks[0];
for (block_id = 0; block_id < PERFCOUNTER_BLOCKID__MAX; block_id++) {
rc = get_block_properties(NodeId, block_id, &block);
rc = hsakmt_get_block_properties(NodeId, block_id, &block);
if (rc != HSAKMT_STATUS_SUCCESS) {
free(counter_props[NodeId]);
counter_props[NodeId] = NULL;
@@ -359,7 +359,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPmcRegisterTrace(HSAuint32 NodeId,
if (!Counters || !TraceRoot || NumberOfCounters == 0)
return HSAKMT_STATUS_INVALID_PARAMETER;
if (validate_nodeid(NodeId, &gpu_id) != HSAKMT_STATUS_SUCCESS)
if (hsakmt_validate_nodeid(NodeId, &gpu_id) != HSAKMT_STATUS_SUCCESS)
return HSAKMT_STATUS_INVALID_NODE_UNIT;
if (NumberOfCounters > MAX_COUNTERS) {
@@ -486,7 +486,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPmcUnregisterTrace(HSAuint32 NodeId,
if (TraceId == 0)
return HSAKMT_STATUS_INVALID_PARAMETER;
if (validate_nodeid(NodeId, &gpu_id) != HSAKMT_STATUS_SUCCESS)
if (hsakmt_validate_nodeid(NodeId, &gpu_id) != HSAKMT_STATUS_SUCCESS)
return HSAKMT_STATUS_INVALID_NODE_UNIT;
trace = (struct perf_trace *)PORT_UINT64_TO_VPTR(TraceId);
@@ -527,7 +527,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPmcAcquireTraceAccess(HSAuint32 NodeId,
if (trace->magic4cc != HSA_PERF_MAGIC4CC)
return HSAKMT_STATUS_INVALID_HANDLE;
if (validate_nodeid(NodeId, &gpu_id) != HSAKMT_STATUS_SUCCESS)
if (hsakmt_validate_nodeid(NodeId, &gpu_id) != HSAKMT_STATUS_SUCCESS)
return HSAKMT_STATUS_INVALID_NODE_UNIT;
return ret;
projects/rocr-runtime/libhsakmt/src/pmc_table.c
+3 -3
View File
@@ -1958,12 +1958,12 @@ static struct perf_counter_block navi_blocks[PERFCOUNTER_BLOCKID__MAX] = {
},
};
HSAKMT_STATUS get_block_properties(uint32_t node_id,
HSAKMT_STATUS hsakmt_get_block_properties(uint32_t node_id,
enum perf_block_id block_id,
struct perf_counter_block *block)
{
uint32_t gfxv = get_gfxv_by_node_id(node_id);
uint16_t dev_id = get_device_id_by_node_id(node_id);
uint32_t gfxv = hsakmt_get_gfxv_by_node_id(node_id);
uint16_t dev_id = hsakmt_get_device_id_by_node_id(node_id);
if (block_id > PERFCOUNTER_BLOCKID__MAX ||
block_id < PERFCOUNTER_BLOCKID__FIRST)
projects/rocr-runtime/libhsakmt/src/pmc_table.h
+1 -1
View File
@@ -67,7 +67,7 @@ struct perf_counter_block {
uint64_t counter_mask;
};
HSAKMT_STATUS get_block_properties(uint32_t node_id,
HSAKMT_STATUS hsakmt_get_block_properties(uint32_t node_id,
enum perf_block_id block_id,
struct perf_counter_block *block);
projects/rocr-runtime/libhsakmt/src/queues.c
+34 -34
View File
@@ -41,7 +41,7 @@
#define DOORBELLS_PAGE_SIZE(ds) (1024 * (ds))
#define WG_CONTEXT_DATA_SIZE_PER_CU(gfxv) \
(get_vgpr_size_per_cu(gfxv) + SGPR_SIZE_PER_CU + \
(hsakmt_get_vgpr_size_per_cu(gfxv) + SGPR_SIZE_PER_CU + \
LDS_SIZE_PER_CU + HWREG_SIZE_PER_CU)
#define CNTL_STACK_BYTES_PER_WAVE(gfxv) \
@@ -84,7 +84,7 @@ struct process_doorbells {
static unsigned int num_doorbells;
static struct process_doorbells *doorbells;
uint32_t get_vgpr_size_per_cu(uint32_t gfxv)
uint32_t hsakmt_get_vgpr_size_per_cu(uint32_t gfxv)
{
uint32_t vgpr_size = 0x40000;
@@ -102,7 +102,7 @@ uint32_t get_vgpr_size_per_cu(uint32_t gfxv)
return vgpr_size;
}
HSAKMT_STATUS init_process_doorbells(unsigned int NumNodes)
HSAKMT_STATUS hsakmt_init_process_doorbells(unsigned int NumNodes)
{
unsigned int i;
HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS;
@@ -133,8 +133,8 @@ static void get_doorbell_map_info(uint32_t node_id,
* GPUVM doorbell on Tonga requires a workaround for VM TLB ACTIVE bit
* lookup bug. Remove ASIC check when this is implemented in amdgpu.
*/
uint32_t gfxv = get_gfxv_by_node_id(node_id);
doorbell->use_gpuvm = (is_dgpu && gfxv != GFX_VERSION_TONGA);
uint32_t gfxv = hsakmt_get_gfxv_by_node_id(node_id);
doorbell->use_gpuvm = (hsakmt_is_dgpu && gfxv != GFX_VERSION_TONGA);
doorbell->size = DOORBELLS_PAGE_SIZE(DOORBELL_SIZE(gfxv));
if (doorbell->size < (uint32_t) PAGE_SIZE) {
@@ -144,7 +144,7 @@ static void get_doorbell_map_info(uint32_t node_id,
return;
}
void destroy_process_doorbells(void)
void hsakmt_destroy_process_doorbells(void)
{
unsigned int i;
@@ -156,8 +156,8 @@ void destroy_process_doorbells(void)
continue;
if (doorbells[i].use_gpuvm) {
fmm_unmap_from_gpu(doorbells[i].mapping);
fmm_release(doorbells[i].mapping);
hsakmt_fmm_unmap_from_gpu(doorbells[i].mapping);
hsakmt_fmm_release(doorbells[i].mapping);
} else
munmap(doorbells[i].mapping, doorbells[i].size);
}
@@ -170,7 +170,7 @@ void destroy_process_doorbells(void)
/* This is a special funcion that should be called only from the child process
* after a fork(). This will clear doorbells duplicated from the parent.
*/
void clear_process_doorbells(void)
void hsakmt_clear_process_doorbells(void)
{
unsigned int i;
@@ -196,7 +196,7 @@ static HSAKMT_STATUS map_doorbell_apu(HSAuint32 NodeId, HSAuint32 gpu_id,
void *ptr;
ptr = mmap(0, doorbells[NodeId].size, PROT_READ|PROT_WRITE,
MAP_SHARED, kfd_fd, doorbell_mmap_offset);
MAP_SHARED, hsakmt_kfd_fd, doorbell_mmap_offset);
if (ptr == MAP_FAILED)
return HSAKMT_STATUS_ERROR;
@@ -211,15 +211,15 @@ static HSAKMT_STATUS map_doorbell_dgpu(HSAuint32 NodeId, HSAuint32 gpu_id,
{
void *ptr;
ptr = fmm_allocate_doorbell(gpu_id, doorbells[NodeId].size,
ptr = hsakmt_fmm_allocate_doorbell(gpu_id, doorbells[NodeId].size,
doorbell_mmap_offset);
if (!ptr)
return HSAKMT_STATUS_ERROR;
/* map for GPU access */
if (fmm_map_to_gpu(ptr, doorbells[NodeId].size, NULL)) {
fmm_release(ptr);
if (hsakmt_fmm_map_to_gpu(ptr, doorbells[NodeId].size, NULL)) {
hsakmt_fmm_release(ptr);
return HSAKMT_STATUS_ERROR;
}
@@ -316,7 +316,7 @@ static bool update_ctx_save_restore_size(uint32_t nodeid, struct queue *q)
return false;
}
void *allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, uint32_t gpu_id,
void *hsakmt_allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, uint32_t gpu_id,
uint32_t NodeId, bool nonPaged,
bool DeviceLocal,
bool Uncached)
@@ -336,8 +336,8 @@ void *allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, uint32_t g
size = ALIGN_UP(size, align);
if (DeviceLocal && !zfb_support)
mem = fmm_allocate_device(gpu_id, NodeId, mem, size, 0, flags);
if (DeviceLocal && !hsakmt_zfb_support)
mem = hsakmt_fmm_allocate_device(gpu_id, NodeId, mem, size, 0, flags);
else {
/* VRAM under ZFB mode should be supported here without any
* additional code
@@ -346,13 +346,13 @@ void *allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, uint32_t g
* nonPaged=0 system memory allocation uses GTT path
*/
if (!nonPaged) {
cpu_id = get_direct_link_cpu(NodeId);
cpu_id = hsakmt_get_direct_link_cpu(NodeId);
if (cpu_id == INVALID_NODEID) {
flags.ui32.NoNUMABind = 1;
cpu_id = 0;
}
}
mem = fmm_allocate_host(gpu_id, cpu_id, mem, size, 0, flags);
mem = hsakmt_fmm_allocate_host(gpu_id, cpu_id, mem, size, 0, flags);
}
if (!mem) {
@@ -383,7 +383,7 @@ void *allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, uint32_t g
return mem;
}
void free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align)
void hsakmt_free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align)
{
size = ALIGN_UP(size, align);
@@ -403,7 +403,7 @@ static void *allocate_exec_aligned_memory(uint32_t size,
bool Uncached)
{
if (!use_ats)
return allocate_exec_aligned_memory_gpu(size, PAGE_SIZE, gpu_id, NodeId,
return hsakmt_allocate_exec_aligned_memory_gpu(size, PAGE_SIZE, gpu_id, NodeId,
nonPaged, DeviceLocal,
Uncached);
return allocate_exec_aligned_memory_cpu(size);
@@ -413,7 +413,7 @@ static void free_exec_aligned_memory(void *addr, uint32_t size, uint32_t align,
bool use_ats)
{
if (!use_ats)
free_exec_aligned_memory_gpu(addr, size, align);
hsakmt_free_exec_aligned_memory_gpu(addr, size, align);
else
munmap(addr, size);
}
@@ -534,13 +534,13 @@ static int handle_concrete_asic(struct queue *q,
/* Allocate unified memory for context save restore
* area on dGPU.
*/
if (!q->use_ats && is_svm_api_supported) {
if (!q->use_ats && hsakmt_is_svm_api_supported) {
uint32_t size = PAGE_ALIGN_UP(q->total_mem_alloc_size);
void *addr;
HSAKMT_STATUS r = HSAKMT_STATUS_ERROR;
pr_info("Allocating GTT for CWSR\n");
addr = mmap_allocate_aligned(PROT_READ | PROT_WRITE,
addr = hsakmt_mmap_allocate_aligned(PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_PRIVATE,
size, GPU_HUGE_PAGE_SIZE, 0,
0, (void *)LONG_MAX);
@@ -634,7 +634,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueueExt(HSAuint32 NodeId,
Priority > HSA_QUEUE_PRIORITY_MAXIMUM)
return HSAKMT_STATUS_INVALID_PARAMETER;
result = validate_nodeid(NodeId, &gpu_id);
result = hsakmt_validate_nodeid(NodeId, &gpu_id);
if (result != HSAKMT_STATUS_SUCCESS)
return result;
@@ -645,7 +645,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueueExt(HSAuint32 NodeId,
memset(q, 0, sizeof(*q));
q->gfxv = get_gfxv_by_node_id(NodeId);
q->gfxv = hsakmt_get_gfxv_by_node_id(NodeId);
q->use_ats = false;
if (q->gfxv == GFX_VERSION_TONGA)
@@ -711,7 +711,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueueExt(HSAuint32 NodeId,
args.queue_priority = priority_map[Priority+3];
args.sdma_engine_id = SdmaEngineId;
err = kmtIoctl(kfd_fd, AMDKFD_IOC_CREATE_QUEUE, &args);
err = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_CREATE_QUEUE, &args);
if (err == -1) {
free_queue(q);
@@ -776,7 +776,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtUpdateQueue(HSA_QUEUEID QueueId,
arg.queue_percentage = QueuePercentage;
arg.queue_priority = priority_map[Priority+3];
int err = kmtIoctl(kfd_fd, AMDKFD_IOC_UPDATE_QUEUE, &arg);
int err = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_UPDATE_QUEUE, &arg);
if (err == -1)
return HSAKMT_STATUS_ERROR;
@@ -796,7 +796,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDestroyQueue(HSA_QUEUEID QueueId)
args.queue_id = q->queue_id;
int err = kmtIoctl(kfd_fd, AMDKFD_IOC_DESTROY_QUEUE, &args);
int err = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_DESTROY_QUEUE, &args);
if (err == -1) {
pr_err("Failed to destroy queue: %s\n", strerror(errno));
@@ -823,7 +823,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetQueueCUMask(HSA_QUEUEID QueueId,
args.num_cu_mask = CUMaskCount;
args.cu_mask_ptr = (uintptr_t)QueueCUMask;
int err = kmtIoctl(kfd_fd, AMDKFD_IOC_SET_CU_MASK, &args);
int err = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_SET_CU_MASK, &args);
if (err == -1)
return HSAKMT_STATUS_ERROR;
@@ -855,7 +855,7 @@ hsaKmtGetQueueInfo(
args.queue_id = q->queue_id;
args.ctl_stack_address = (uintptr_t)q->ctx_save_restore;
if (kmtIoctl(kfd_fd, AMDKFD_IOC_GET_QUEUE_WAVE_STATE, &args) < 0)
if (hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_GET_QUEUE_WAVE_STATE, &args) < 0)
return HSAKMT_STATUS_ERROR;
QueueInfo->ControlStackTop = (void *)(args.ctl_stack_address +
@@ -885,7 +885,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetTrapHandler(HSAuint32 Node,
CHECK_KFD_OPEN();
result = validate_nodeid(Node, &gpu_id);
result = hsakmt_validate_nodeid(Node, &gpu_id);
if (result != HSAKMT_STATUS_SUCCESS)
return result;
@@ -893,12 +893,12 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetTrapHandler(HSAuint32 Node,
args.tba_addr = (uintptr_t)TrapHandlerBaseAddress;
args.tma_addr = (uintptr_t)TrapBufferBaseAddress;
int err = kmtIoctl(kfd_fd, AMDKFD_IOC_SET_TRAP_HANDLER, &args);
int err = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_SET_TRAP_HANDLER, &args);
return (err == -1) ? HSAKMT_STATUS_ERROR : HSAKMT_STATUS_SUCCESS;
}
uint32_t *convert_queue_ids(HSAuint32 NumQueues, HSA_QUEUEID *Queues)
uint32_t *hsakmt_convert_queue_ids(HSAuint32 NumQueues, HSA_QUEUEID *Queues)
{
uint32_t *queue_ids_ptr;
unsigned int i;
@@ -930,7 +930,7 @@ hsaKmtAllocQueueGWS(
args.queue_id = (HSAuint32)q->queue_id;
args.num_gws = nGWS;
int err = kmtIoctl(kfd_fd, AMDKFD_IOC_ALLOC_QUEUE_GWS, &args);
int err = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_ALLOC_QUEUE_GWS, &args);
if (!err && firstGWS)
*firstGWS = args.first_gws;
projects/rocr-runtime/libhsakmt/src/rbtree.c
+6 -6
View File
@@ -33,7 +33,7 @@ static inline void rbtree_right_rotate(rbtree_node_t **root,
rbtree_node_t *sentinel, rbtree_node_t *node);
static void
rbtree_insert_value(rbtree_node_t *temp, rbtree_node_t *node,
hsakmt_rbtree_insert_value(rbtree_node_t *temp, rbtree_node_t *node,
rbtree_node_t *sentinel)
{
rbtree_node_t **p;
@@ -59,7 +59,7 @@ rbtree_insert_value(rbtree_node_t *temp, rbtree_node_t *node,
void
rbtree_insert(rbtree_t *tree, rbtree_node_t *node)
hsakmt_rbtree_insert(rbtree_t *tree, rbtree_node_t *node)
{
rbtree_node_t **root, *temp, *sentinel;
@@ -78,7 +78,7 @@ rbtree_insert(rbtree_t *tree, rbtree_node_t *node)
return;
}
rbtree_insert_value(*root, node, sentinel);
hsakmt_rbtree_insert_value(*root, node, sentinel);
/* re-balance tree */
@@ -131,7 +131,7 @@ rbtree_insert(rbtree_t *tree, rbtree_node_t *node)
void
rbtree_delete(rbtree_t *tree, rbtree_node_t *node)
hsakmt_rbtree_delete(rbtree_t *tree, rbtree_node_t *node)
{
unsigned int red;
rbtree_node_t **root, *sentinel, *subst, *temp, *w;
@@ -346,7 +346,7 @@ rbtree_right_rotate(rbtree_node_t **root, rbtree_node_t *sentinel,
rbtree_node_t *
rbtree_next(rbtree_t *tree, rbtree_node_t *node)
hsakmt_rbtree_next(rbtree_t *tree, rbtree_node_t *node)
{
rbtree_node_t *root, *sentinel, *parent;
@@ -374,7 +374,7 @@ rbtree_next(rbtree_t *tree, rbtree_node_t *node)
}
rbtree_node_t *
rbtree_prev(rbtree_t *tree, rbtree_node_t *node)
hsakmt_rbtree_prev(rbtree_t *tree, rbtree_node_t *node)
{
rbtree_node_t *root, *sentinel, *parent;
projects/rocr-runtime/libhsakmt/src/rbtree.h
+4 -4
View File
@@ -62,11 +62,11 @@ struct rbtree_s {
rbtree_sentinel_init(&(tree)->sentinel); \
(tree)->root = &(tree)->sentinel;
void rbtree_insert(rbtree_t *tree, rbtree_node_t *node);
void rbtree_delete(rbtree_t *tree, rbtree_node_t *node);
rbtree_node_t *rbtree_prev(rbtree_t *tree,
void hsakmt_rbtree_insert(rbtree_t *tree, rbtree_node_t *node);
void hsakmt_rbtree_delete(rbtree_t *tree, rbtree_node_t *node);
rbtree_node_t *hsakmt_rbtree_prev(rbtree_t *tree,
rbtree_node_t *node);
rbtree_node_t *rbtree_next(rbtree_t *tree,
rbtree_node_t *hsakmt_rbtree_next(rbtree_t *tree,
rbtree_node_t *node);
#define rbt_red(node) ((node)->color = 1)
projects/rocr-runtime/libhsakmt/src/spm.c
+6 -6
View File
@@ -35,7 +35,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSPMAcquire(HSAuint32 PreferredNode)
struct kfd_ioctl_spm_args args = {0};
uint32_t gpu_id;
ret = validate_nodeid(PreferredNode, &gpu_id);
ret = hsakmt_validate_nodeid(PreferredNode, &gpu_id);
if (ret != HSAKMT_STATUS_SUCCESS) {
pr_err("[%s] invalid node ID: %d\n", __func__, PreferredNode);
return ret;
@@ -45,7 +45,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSPMAcquire(HSAuint32 PreferredNode)
args.op = KFD_IOCTL_SPM_OP_ACQUIRE;
args.gpu_id = gpu_id;
ret = kmtIoctl(kfd_fd, AMDKFD_IOC_RLC_SPM, &args);
ret = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_RLC_SPM, &args);
return ret;
}
@@ -63,7 +63,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSPMSetDestBuffer(HSAuint32 PreferredNode,
ret = HSAKMT_STATUS_SUCCESS;
ret = validate_nodeid(PreferredNode, &gpu_id);
ret = hsakmt_validate_nodeid(PreferredNode, &gpu_id);
args.timeout = *timeout;
args.dest_buf = (uint64_t)DestMemoryAddress;
@@ -71,7 +71,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSPMSetDestBuffer(HSAuint32 PreferredNode,
args.op = KFD_IOCTL_SPM_OP_SET_DEST_BUF;
args.gpu_id = gpu_id;
ret = kmtIoctl(kfd_fd, AMDKFD_IOC_RLC_SPM, &args);
ret = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_RLC_SPM, &args);
*SizeCopied = args.bytes_copied;
*isSPMDataLoss = args.has_data_loss;
@@ -86,7 +86,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSPMRelease(HSAuint32 PreferredNode)
struct kfd_ioctl_spm_args args = {0};
uint32_t gpu_id;
ret = validate_nodeid(PreferredNode, &gpu_id);
ret = hsakmt_validate_nodeid(PreferredNode, &gpu_id);
if (ret != HSAKMT_STATUS_SUCCESS) {
pr_err("[%s] invalid node ID: %d\n", __func__, PreferredNode);
return ret;
@@ -95,7 +95,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSPMRelease(HSAuint32 PreferredNode)
args.op = KFD_IOCTL_SPM_OP_RELEASE;
args.gpu_id = gpu_id;
ret = kmtIoctl(kfd_fd, AMDKFD_IOC_RLC_SPM, &args);
ret = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_RLC_SPM, &args);
return ret;
}
projects/rocr-runtime/libhsakmt/src/svm.c
+7 -7
View File
@@ -80,7 +80,7 @@ hsaKmtSVMSetAttr(void *start_addr, HSAuint64 size, unsigned int nattr,
continue;
}
r = validate_nodeid(attrs[i].value, &args->attrs[i].value);
r = hsakmt_validate_nodeid(attrs[i].value, &args->attrs[i].value);
if (r != HSAKMT_STATUS_SUCCESS) {
pr_debug("invalid node ID: %d\n", attrs[i].value);
return r;
@@ -94,7 +94,7 @@ hsaKmtSVMSetAttr(void *start_addr, HSAuint64 size, unsigned int nattr,
}
/* Driver does one copy_from_user, with extra attrs size */
r = kmtIoctl(kfd_fd, AMDKFD_IOC_SVM + (s_attr << _IOC_SIZESHIFT), args);
r = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_SVM + (s_attr << _IOC_SIZESHIFT), args);
if (r) {
pr_debug("op set range attrs failed %s\n", strerror(errno));
return HSAKMT_STATUS_ERROR;
@@ -139,7 +139,7 @@ hsaKmtSVMGetAttr(void *start_addr, HSAuint64 size, unsigned int nattr,
attrs[i].type != KFD_IOCTL_SVM_ATTR_NO_ACCESS)
continue;
r = validate_nodeid(attrs[i].value, &args->attrs[i].value);
r = hsakmt_validate_nodeid(attrs[i].value, &args->attrs[i].value);
if (r != HSAKMT_STATUS_SUCCESS) {
pr_debug("invalid node ID: %d\n", attrs[i].value);
return r;
@@ -150,7 +150,7 @@ hsaKmtSVMGetAttr(void *start_addr, HSAuint64 size, unsigned int nattr,
}
/* Driver does one copy_from_user, with extra attrs size */
r = kmtIoctl(kfd_fd, AMDKFD_IOC_SVM + (s_attr << _IOC_SIZESHIFT), args);
r = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_SVM + (s_attr << _IOC_SIZESHIFT), args);
if (r) {
pr_debug("op get range attrs failed %s\n", strerror(errno));
return HSAKMT_STATUS_ERROR;
@@ -174,7 +174,7 @@ hsaKmtSVMGetAttr(void *start_addr, HSAuint64 size, unsigned int nattr,
attrs[i].value = INVALID_NODEID;
break;
default:
r = gpuid_to_nodeid(attrs[i].value, &attrs[i].value);
r = hsakmt_gpuid_to_nodeid(attrs[i].value, &attrs[i].value);
if (r != HSAKMT_STATUS_SUCCESS) {
pr_debug("invalid GPU ID: %d\n",
attrs[i].value);
@@ -196,13 +196,13 @@ hsaKmtSetGetXNACKMode(HSAint32 * enable)
args.xnack_enabled = *enable;
if (kmtIoctl(kfd_fd, AMDKFD_IOC_SET_XNACK_MODE, &args)) {
if (hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_SET_XNACK_MODE, &args)) {
if (errno == EPERM) {
pr_debug("set mode not supported %s\n",
strerror(errno));
return HSAKMT_STATUS_NOT_SUPPORTED;
} else if (errno == EBUSY) {
pr_debug("kmtIoctl queues not empty %s\n",
pr_debug("hsakmt_ioctl queues not empty %s\n",
strerror(errno));
}
return HSAKMT_STATUS_ERROR;
projects/rocr-runtime/libhsakmt/src/time.c
+2 -2
View File
@@ -36,13 +36,13 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetClockCounters(HSAuint32 NodeId,
CHECK_KFD_OPEN();
result = validate_nodeid(NodeId, &gpu_id);
result = hsakmt_validate_nodeid(NodeId, &gpu_id);
if (result != HSAKMT_STATUS_SUCCESS)
return result;
args.gpu_id = gpu_id;
err = kmtIoctl(kfd_fd, AMDKFD_IOC_GET_CLOCK_COUNTERS, &args);
err = hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_GET_CLOCK_COUNTERS, &args);
if (err < 0) {
result = HSAKMT_STATUS_ERROR;
} else {
projects/rocr-runtime/libhsakmt/src/topology.c
+39 -39
View File
@@ -695,7 +695,7 @@ static HSAKMT_STATUS topology_sysfs_check_node_supported(uint32_t sysfs_node_id,
}
/* Open DRM Render device */
ret_value = open_drm_render_device(drm_render_minor);
ret_value = hsakmt_open_drm_render_device(drm_render_minor);
if (ret_value > 0)
*is_node_supported = true;
else if (ret_value != -ENOENT && ret_value != -EPERM)
@@ -707,7 +707,7 @@ err:
return ret;
}
HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props)
HSAKMT_STATUS hsakmt_topology_sysfs_get_system_props(HsaSystemProperties *props)
{
FILE *fd;
char *read_buf, *p;
@@ -818,16 +818,16 @@ static const struct hsa_gfxip_table *find_hsa_gfxip_device(uint16_t device_id, u
return NULL;
}
void topology_setup_is_dgpu_param(HsaNodeProperties *props)
void hsakmt_topology_setup_is_dgpu_param(HsaNodeProperties *props)
{
/* if we found a dGPU node, then treat the whole system as dGPU */
if (!props->NumCPUCores && props->NumFComputeCores)
is_dgpu = true;
hsakmt_is_dgpu = true;
}
bool topology_is_svm_needed(HSA_ENGINE_ID EngineId)
bool hsakmt_topology_is_svm_needed(HSA_ENGINE_ID EngineId)
{
if (is_dgpu)
if (hsakmt_is_dgpu)
return true;
if (HSA_GET_GFX_VERSION_FULL(EngineId.ui32) >= GFX_VERSION_VEGA10)
@@ -1195,7 +1195,7 @@ static HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id,
gfxv = (uint32_t)prop_val;
}
if (!is_svm_api_supported)
if (!hsakmt_is_svm_api_supported)
props->Capability.ui32.SVMAPISupported = 0;
/* Bail out early, if a CPU node */
@@ -1255,7 +1255,7 @@ static HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id,
/* Get VGPR/SGPR size in byte per CU */
props->SGPRSizePerCU = SGPR_SIZE_PER_CU;
props->VGPRSizePerCU = get_vgpr_size_per_cu(HSA_GET_GFX_VERSION_FULL(props->EngineId.ui32));
props->VGPRSizePerCU = hsakmt_get_vgpr_size_per_cu(HSA_GET_GFX_VERSION_FULL(props->EngineId.ui32));
} else if (props->DeviceId)
/* still return success */
@@ -1957,7 +1957,7 @@ retry:
ret = topology_sysfs_get_generation(&gen_start);
if (ret != HSAKMT_STATUS_SUCCESS)
goto err;
ret = topology_sysfs_get_system_props(&sys_props);
ret = hsakmt_topology_sysfs_get_system_props(&sys_props);
if (ret != HSAKMT_STATUS_SUCCESS)
goto err;
if (sys_props.NumNodes > 0) {
@@ -2135,7 +2135,7 @@ void topology_drop_snapshot(void)
}
}
HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id)
HSAKMT_STATUS hsakmt_validate_nodeid(uint32_t nodeid, uint32_t *gpu_id)
{
if (!g_props || !g_system || g_system->NumNodes <= nodeid)
return HSAKMT_STATUS_INVALID_NODE_UNIT;
@@ -2145,7 +2145,7 @@ HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id)
return HSAKMT_STATUS_SUCCESS;
}
HSAKMT_STATUS gpuid_to_nodeid(uint32_t gpu_id, uint32_t *node_id)
HSAKMT_STATUS hsakmt_gpuid_to_nodeid(uint32_t gpu_id, uint32_t *node_id)
{
uint64_t node_idx;
@@ -2185,11 +2185,11 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAcquireSystemProperties(HsaSystemProperties *Syste
assert(g_system);
err = fmm_init_process_apertures(g_system->NumNodes);
err = hsakmt_fmm_init_process_apertures(g_system->NumNodes);
if (err != HSAKMT_STATUS_SUCCESS)
goto init_process_apertures_failed;
err = init_process_doorbells(g_system->NumNodes);
err = hsakmt_init_process_doorbells(g_system->NumNodes);
if (err != HSAKMT_STATUS_SUCCESS)
goto init_doorbells_failed;
@@ -2198,7 +2198,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAcquireSystemProperties(HsaSystemProperties *Syste
goto out;
init_doorbells_failed:
fmm_destroy_process_apertures();
hsakmt_fmm_destroy_process_apertures();
init_process_apertures_failed:
topology_drop_snapshot();
@@ -2211,8 +2211,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtReleaseSystemProperties(void)
{
pthread_mutex_lock(&hsakmt_mutex);
destroy_process_doorbells();
fmm_destroy_process_apertures();
hsakmt_destroy_process_doorbells();
hsakmt_fmm_destroy_process_apertures();
topology_drop_snapshot();
pthread_mutex_unlock(&hsakmt_mutex);
@@ -2220,7 +2220,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtReleaseSystemProperties(void)
return HSAKMT_STATUS_SUCCESS;
}
HSAKMT_STATUS topology_get_node_props(HSAuint32 NodeId,
HSAKMT_STATUS hsakmt_topology_get_node_props(HSAuint32 NodeId,
HsaNodeProperties *NodeProperties)
{
if (!g_system || !g_props || NodeId >= g_system->NumNodes)
@@ -2242,21 +2242,21 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeProperties(HSAuint32 NodeId,
CHECK_KFD_OPEN();
pthread_mutex_lock(&hsakmt_mutex);
err = validate_nodeid(NodeId, &gpu_id);
err = hsakmt_validate_nodeid(NodeId, &gpu_id);
if (err != HSAKMT_STATUS_SUCCESS)
goto out;
err = topology_get_node_props(NodeId, NodeProperties);
err = hsakmt_topology_get_node_props(NodeId, NodeProperties);
if (err != HSAKMT_STATUS_SUCCESS)
goto out;
/* For CPU only node don't add any additional GPU memory banks. */
if (gpu_id) {
uint64_t base, limit;
if (is_dgpu)
if (hsakmt_is_dgpu)
NodeProperties->NumMemoryBanks += NUM_OF_DGPU_HEAPS;
else
NodeProperties->NumMemoryBanks += NUM_OF_IGPU_HEAPS;
if (fmm_get_aperture_base_and_limit(FMM_MMIO, gpu_id, &base,
if (hsakmt_fmm_get_aperture_base_and_limit(FMM_MMIO, gpu_id, &base,
&limit) == HSAKMT_STATUS_SUCCESS)
NodeProperties->NumMemoryBanks += 1;
}
@@ -2280,7 +2280,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeMemoryProperties(HSAuint32 NodeId,
CHECK_KFD_OPEN();
pthread_mutex_lock(&hsakmt_mutex);
err = validate_nodeid(NodeId, &gpu_id);
err = hsakmt_validate_nodeid(NodeId, &gpu_id);
if (err != HSAKMT_STATUS_SUCCESS)
goto out;
@@ -2297,7 +2297,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeMemoryProperties(HSAuint32 NodeId,
/*Add LDS*/
if (i < NumBanks &&
fmm_get_aperture_base_and_limit(FMM_LDS, gpu_id,
hsakmt_fmm_get_aperture_base_and_limit(FMM_LDS, gpu_id,
&MemoryProperties[i].VirtualBaseAddress, &aperture_limit) == HSAKMT_STATUS_SUCCESS) {
MemoryProperties[i].HeapType = HSA_HEAPTYPE_GPU_LDS;
MemoryProperties[i].SizeInBytes = g_props[NodeId].node.LDSSizeInKB * 1024;
@@ -2308,9 +2308,9 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeMemoryProperties(HSAuint32 NodeId,
* For dGPU the topology node contains Local Memory and it is added by
* the for loop above
*/
if (get_gfxv_by_node_id(NodeId) == GFX_VERSION_KAVERI && i < NumBanks &&
if (hsakmt_get_gfxv_by_node_id(NodeId) == GFX_VERSION_KAVERI && i < NumBanks &&
g_props[NodeId].node.LocalMemSize > 0 &&
fmm_get_aperture_base_and_limit(FMM_GPUVM, gpu_id,
hsakmt_fmm_get_aperture_base_and_limit(FMM_GPUVM, gpu_id,
&MemoryProperties[i].VirtualBaseAddress, &aperture_limit) == HSAKMT_STATUS_SUCCESS) {
MemoryProperties[i].HeapType = HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE;
MemoryProperties[i].SizeInBytes = g_props[NodeId].node.LocalMemSize;
@@ -2319,7 +2319,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeMemoryProperties(HSAuint32 NodeId,
/* Add SCRATCH */
if (i < NumBanks &&
fmm_get_aperture_base_and_limit(FMM_SCRATCH, gpu_id,
hsakmt_fmm_get_aperture_base_and_limit(FMM_SCRATCH, gpu_id,
&MemoryProperties[i].VirtualBaseAddress, &aperture_limit) == HSAKMT_STATUS_SUCCESS) {
MemoryProperties[i].HeapType = HSA_HEAPTYPE_GPU_SCRATCH;
MemoryProperties[i].SizeInBytes = (aperture_limit - MemoryProperties[i].VirtualBaseAddress) + 1;
@@ -2327,8 +2327,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeMemoryProperties(HSAuint32 NodeId,
}
/* Add SVM aperture */
if (topology_is_svm_needed(g_props[NodeId].node.EngineId) && i < NumBanks &&
fmm_get_aperture_base_and_limit(
if (hsakmt_topology_is_svm_needed(g_props[NodeId].node.EngineId) && i < NumBanks &&
hsakmt_fmm_get_aperture_base_and_limit(
FMM_SVM, gpu_id, &MemoryProperties[i].VirtualBaseAddress,
&aperture_limit) == HSAKMT_STATUS_SUCCESS) {
MemoryProperties[i].HeapType = HSA_HEAPTYPE_DEVICE_SVM;
@@ -2338,7 +2338,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeMemoryProperties(HSAuint32 NodeId,
/* Add mmio aperture */
if (i < NumBanks &&
fmm_get_aperture_base_and_limit(FMM_MMIO, gpu_id,
hsakmt_fmm_get_aperture_base_and_limit(FMM_MMIO, gpu_id,
&MemoryProperties[i].VirtualBaseAddress, &aperture_limit) == HSAKMT_STATUS_SUCCESS) {
MemoryProperties[i].HeapType = HSA_HEAPTYPE_MMIO_REMAP;
MemoryProperties[i].SizeInBytes = (aperture_limit - MemoryProperties[i].VirtualBaseAddress) + 1;
@@ -2387,7 +2387,7 @@ out:
return err;
}
HSAKMT_STATUS topology_get_iolink_props(HSAuint32 NodeId,
HSAKMT_STATUS hsakmt_topology_get_iolink_props(HSAuint32 NodeId,
HSAuint32 NumIoLinks,
HsaIoLinkProperties *IoLinkProperties)
{
@@ -2425,19 +2425,19 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeIoLinkProperties(HSAuint32 NodeId,
}
assert(g_props[NodeId].link);
err = topology_get_iolink_props(NodeId, NumIoLinks, IoLinkProperties);
err = hsakmt_topology_get_iolink_props(NodeId, NumIoLinks, IoLinkProperties);
out:
pthread_mutex_unlock(&hsakmt_mutex);
return err;
}
uint32_t get_gfxv_by_node_id(HSAuint32 node_id)
uint32_t hsakmt_get_gfxv_by_node_id(HSAuint32 node_id)
{
return HSA_GET_GFX_VERSION_FULL(g_props[node_id].node.EngineId.ui32);
}
uint16_t get_device_id_by_node_id(HSAuint32 node_id)
uint16_t hsakmt_get_device_id_by_node_id(HSAuint32 node_id)
{
if (!g_props || !g_system || g_system->NumNodes <= node_id)
return 0;
@@ -2445,14 +2445,14 @@ uint16_t get_device_id_by_node_id(HSAuint32 node_id)
return g_props[node_id].node.DeviceId;
}
bool prefer_ats(HSAuint32 node_id)
bool hsakmt_prefer_ats(HSAuint32 node_id)
{
return g_props[node_id].node.Capability.ui32.HSAMMUPresent
&& g_props[node_id].node.NumCPUCores
&& g_props[node_id].node.NumFComputeCores;
}
uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id)
uint16_t hsakmt_get_device_id_by_gpu_id(HSAuint32 gpu_id)
{
unsigned int i;
@@ -2467,7 +2467,7 @@ uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id)
return 0;
}
uint32_t get_direct_link_cpu(uint32_t gpu_node)
uint32_t hsakmt_get_direct_link_cpu(uint32_t gpu_node)
{
HSAuint64 size = 0;
int32_t cpu_id;
@@ -2486,7 +2486,7 @@ uint32_t get_direct_link_cpu(uint32_t gpu_node)
}
HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array,
HSAKMT_STATUS hsakmt_validate_nodeid_array(uint32_t **gpu_id_array,
uint32_t NumberOfNodes, uint32_t *NodeArray)
{
HSAKMT_STATUS ret;
@@ -2500,7 +2500,7 @@ HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array,
if (!(*gpu_id_array))
return HSAKMT_STATUS_NO_MEMORY;
for (i = 0; i < NumberOfNodes; i++) {
ret = validate_nodeid(NodeArray[i], *gpu_id_array + i);
ret = hsakmt_validate_nodeid(NodeArray[i], *gpu_id_array + i);
if (ret != HSAKMT_STATUS_SUCCESS) {
free(*gpu_id_array);
break;
@@ -2510,7 +2510,7 @@ HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array,
return ret;
}
inline uint32_t get_num_sysfs_nodes(void)
inline uint32_t hsakmt_get_num_sysfs_nodes(void)
{
return num_sysfs_nodes;
}
projects/rocr-runtime/libhsakmt/src/version.c
+6 -6
View File
@@ -28,26 +28,26 @@
#include <string.h>
#include "hsakmt/linux/kfd_ioctl.h"
HsaVersionInfo kfd_version_info;
HsaVersionInfo hsakmt_kfd_version_info;
HSAKMT_STATUS HSAKMTAPI hsaKmtGetVersion(HsaVersionInfo *VersionInfo)
{
CHECK_KFD_OPEN();
*VersionInfo = kfd_version_info;
*VersionInfo = hsakmt_kfd_version_info;
return HSAKMT_STATUS_SUCCESS;
}
HSAKMT_STATUS init_kfd_version(void)
HSAKMT_STATUS hsakmt_init_kfd_version(void)
{
struct kfd_ioctl_get_version_args args = {0};
if (kmtIoctl(kfd_fd, AMDKFD_IOC_GET_VERSION, &args) == -1)
if (hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_GET_VERSION, &args) == -1)
return HSAKMT_STATUS_ERROR;
kfd_version_info.KernelInterfaceMajorVersion = args.major_version;
kfd_version_info.KernelInterfaceMinorVersion = args.minor_version;
hsakmt_kfd_version_info.KernelInterfaceMajorVersion = args.major_version;
hsakmt_kfd_version_info.KernelInterfaceMinorVersion = args.minor_version;
if (args.major_version != 1)
return HSAKMT_STATUS_DRIVER_MISMATCH;
projects/rocr-runtime/libhsakmt/tests/kfdtest/src/Dispatch.cpp
+3 -3
View File
@@ -169,14 +169,14 @@ void Dispatch::BuildIb() {
const unsigned int COMPUTE_PGM_VALUES_GFX8[] = {
static_cast<uint32_t>(shiftedIsaAddr), // PGM_LO
static_cast<uint32_t>(shiftedIsaAddr >> 32) // PGM_HI
| (is_dgpu() ? 0 : (1<<8)) // including PGM_ATC=?
| (hsakmt_is_dgpu() ? 0 : (1<<8)) // including PGM_ATC=?
};
// Starts at COMPUTE_PGM_LO
const unsigned int COMPUTE_PGM_VALUES_GFX9[] = {
static_cast<uint32_t>(shiftedIsaAddr), // PGM_LO
static_cast<uint32_t>(shiftedIsaAddr >> 32) // PGM_HI
| (is_dgpu() ? 0 : (1<<8)), // including PGM_ATC=?
| (hsakmt_is_dgpu() ? 0 : (1<<8)), // including PGM_ATC=?
0,
0,
static_cast<uint32_t>(m_scratch_base >> 8), // compute_dispatch_scratch_base
@@ -222,7 +222,7 @@ void Dispatch::BuildIb() {
0, // COMPUTE_USER_DATA_15 - - unused
};
const unsigned int DISPATCH_INIT_VALUE = 0x00000021 | (is_dgpu() ? 0 : 0x1000) |
const unsigned int DISPATCH_INIT_VALUE = 0x00000021 | (hsakmt_is_dgpu() ? 0 : 0x1000) |
((m_FamilyId >= FAMILY_NV) ? 0x8000 : 0);
// {COMPUTE_SHADER_EN=1, PARTIAL_TG_EN=0, FORCE_START_AT_000=0, ORDERED_APPEND_ENBL=0,
// ORDERED_APPEND_MODE=0, USE_THREAD_DIMENSIONS=1, ORDER_MODE=0, DISPATCH_CACHE_CNTL=0,
projects/rocr-runtime/libhsakmt/tests/kfdtest/src/KFDEvictTest.cpp
+2 -2
View File
@@ -75,7 +75,7 @@ void KFDEvictTest::AllocBuffers(HSAuint32 defaultGPUNode, HSAuint32 count, HSAui
for (HSAuint32 i = 0; i < count; ) {
ret = hsaKmtAllocMemory(defaultGPUNode, vramBufSize, m_Flags, &m_pBuf);
if (ret == HSAKMT_STATUS_SUCCESS) {
if (is_dgpu()) {
if (hsakmt_is_dgpu()) {
if (hsaKmtMapMemoryToGPUNodes(m_pBuf, vramBufSize, NULL,
mapFlags, 1, reinterpret_cast<HSAuint32 *>(&defaultGPUNode)) == HSAKMT_STATUS_ERROR) {
EXPECT_SUCCESS(hsaKmtFreeMemory(m_pBuf, vramBufSize));
@@ -103,7 +103,7 @@ void KFDEvictTest::FreeBuffers(std::vector<void *> &pBuffers, HSAuint64 vramBufS
for (HSAuint32 i = 0; i < pBuffers.size(); i++) {
m_pBuf = pBuffers[i];
if (m_pBuf != NULL) {
if (is_dgpu())
if (hsakmt_is_dgpu())
EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(m_pBuf));
EXPECT_SUCCESS(hsaKmtFreeMemory(m_pBuf, vramBufSize));
}
projects/rocr-runtime/libhsakmt/tests/kfdtest/src/KFDExceptionTest.cpp
+5 -5
View File
@@ -197,7 +197,7 @@ TEST_F(KFDExceptionTest, AddressFault) {
int childStatus;
waitpid(m_ChildPid, &childStatus, 0);
if (is_dgpu()) {
if (hsakmt_is_dgpu()) {
EXPECT_EQ(WIFEXITED(childStatus), true);
EXPECT_EQ(WEXITSTATUS(childStatus), HSAKMT_STATUS_SUCCESS);
} else {
@@ -242,7 +242,7 @@ TEST_F(KFDExceptionTest, PermissionFault) {
int childStatus;
waitpid(m_ChildPid, &childStatus, 0);
if (is_dgpu()) {
if (hsakmt_is_dgpu()) {
EXPECT_EQ(WIFEXITED(childStatus), true);
EXPECT_EQ(WEXITSTATUS(childStatus), HSAKMT_STATUS_SUCCESS);
} else {
@@ -289,7 +289,7 @@ TEST_F(KFDExceptionTest, PermissionFaultUserPointer) {
int childStatus;
waitpid(m_ChildPid, &childStatus, 0);
if (is_dgpu()) {
if (hsakmt_is_dgpu()) {
EXPECT_EQ(WIFEXITED(childStatus), true);
EXPECT_EQ(WEXITSTATUS(childStatus), HSAKMT_STATUS_SUCCESS);
} else {
@@ -328,7 +328,7 @@ TEST_F(KFDExceptionTest, FaultStorm) {
int childStatus;
waitpid(m_ChildPid, &childStatus, 0);
if (is_dgpu()) {
if (hsakmt_is_dgpu()) {
EXPECT_EQ(WIFEXITED(childStatus), true);
EXPECT_EQ(WEXITSTATUS(childStatus), HSAKMT_STATUS_SUCCESS);
} else {
@@ -379,7 +379,7 @@ TEST_F(KFDExceptionTest, SdmaQueueException) {
int childStatus;
waitpid(m_ChildPid, &childStatus, 0);
if (is_dgpu()) {
if (hsakmt_is_dgpu()) {
EXPECT_EQ(WIFEXITED(childStatus), true);
EXPECT_EQ(WEXITSTATUS(childStatus), HSAKMT_STATUS_SUCCESS);
} else {
projects/rocr-runtime/libhsakmt/tests/kfdtest/src/KFDGraphicsInterop.cpp
+1 -1
View File
@@ -149,7 +149,7 @@ TEST_F(KFDGraphicsInterop, RegisterGraphicsHandle) {
TEST_F(KFDGraphicsInterop, RegisterForeignDeviceMem) {
TEST_START(TESTPROFILE_RUNALL)
if (!is_dgpu()) {
if (!hsakmt_is_dgpu()) {
LOG() << "Skipping test: Only supported on multi-dGPU system." << std::endl;
return;
}
projects/rocr-runtime/libhsakmt/tests/kfdtest/src/KFDLocalMemoryTest.cpp
+4 -4
View File
@@ -51,7 +51,7 @@ TEST_F(KFDLocalMemoryTest, AccessLocalMem) {
ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node";
/* Skip test if not on dGPU path, which the test depends on */
if (!is_dgpu()) {
if (!hsakmt_is_dgpu()) {
LOG() << "Not dGPU path, skipping the test" << std::endl;
return;
}
@@ -163,7 +163,7 @@ TEST_F(KFDLocalMemoryTest, VerifyContentsAfterUnmapAndMap) {
ASSERT_SUCCESS(queue.Create(defaultGPUNode));
queue.SetSkipWaitConsump(0);
if (!is_dgpu())
if (!hsakmt_is_dgpu())
ASSERT_SUCCESS(hsaKmtMapMemoryToGPUNodes(LocalBuffer.As<void*>(), LocalBuffer.Size(), &AlternateVAGPU,
mapFlags, 1, reinterpret_cast<HSAuint32 *>(&defaultGPUNode)));
@@ -183,7 +183,7 @@ TEST_F(KFDLocalMemoryTest, VerifyContentsAfterUnmapAndMap) {
EXPECT_SUCCESS(queue.Destroy());
EXPECT_EQ(SysBufferB.As<unsigned int*>()[0], 0x01010101);
if (!is_dgpu())
if (!hsakmt_is_dgpu())
EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(LocalBuffer.As<void*>()));
TEST_END
@@ -285,7 +285,7 @@ TEST_F(KFDLocalMemoryTest, Fragmentation) {
* workaround. Also nicely matches the 8x bigger GPUVM address
* space on AMDGPU compared to RADEON.
*/
unsigned pageSize = is_dgpu() ? PAGE_SIZE*8 : PAGE_SIZE;
unsigned pageSize = hsakmt_is_dgpu() ? PAGE_SIZE*8 : PAGE_SIZE;
fbSize /= pageSize;
unsigned maxOrder = 0;
// Limit maxOrder up to 14 so this test doesn't run longer than 10 mins
projects/rocr-runtime/libhsakmt/tests/kfdtest/src/KFDMemoryTest.cpp
+11 -11
View File
@@ -72,7 +72,7 @@ TEST_F(KFDMemoryTest, MMapLarge) {
TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX);
TEST_START(TESTPROFILE_RUNALL)
if (!is_dgpu()) {
if (!hsakmt_is_dgpu()) {
LOG() << "Skipping test: Test not supported on APU." << std::endl;
return;
}
@@ -298,7 +298,7 @@ TEST_F(KFDMemoryTest, AccessPPRMem) {
int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode();
ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node";
if (is_dgpu()) {
if (hsakmt_is_dgpu()) {
LOG() << "Skipping test: Test requires APU." << std::endl;
return;
}
@@ -440,7 +440,7 @@ TEST_F(KFDMemoryTest, MemoryRegister) {
}
TEST_F(KFDMemoryTest, MemoryRegisterSamePtr) {
if (!is_dgpu()) {
if (!hsakmt_is_dgpu()) {
LOG() << "Skipping test: Will run on APU once APU+dGPU supported." << std::endl;
return;
}
@@ -709,7 +709,7 @@ void KFDMemoryTest::SearchLargestBuffer(int allocNode, const HsaMemFlags &memFla
* the onerous memory swap operation. So we limit the buffer size that way.
*/
TEST_F(KFDMemoryTest, LargestSysBufferTest) {
if (!is_dgpu()) {
if (!hsakmt_is_dgpu()) {
LOG() << "Skipping test: Running on APU fails and locks the system." << std::endl;
return;
}
@@ -737,7 +737,7 @@ TEST_F(KFDMemoryTest, LargestSysBufferTest) {
}
TEST_F(KFDMemoryTest, LargestVramBufferTest) {
if (!is_dgpu()) {
if (!hsakmt_is_dgpu()) {
LOG() << "Skipping test: Running on APU fails and locks the system." << std::endl;
return;
}
@@ -783,7 +783,7 @@ TEST_F(KFDMemoryTest, LargestVramBufferTest) {
* performed on each buffer.
*/
TEST_F(KFDMemoryTest, BigSysBufferStressTest) {
if (!is_dgpu()) {
if (!hsakmt_is_dgpu()) {
LOG() << "Skipping test: Running on APU fails and locks the system." << std::endl;
return;
}
@@ -1103,7 +1103,7 @@ TEST_F(KFDMemoryTest, QueryPointerInfo) {
EXPECT_EQ(ptrInfo.GPUAddress, (HSAuint64)hostBuffer.As<void*>());
EXPECT_EQ(ptrInfo.SizeInBytes, (HSAuint64)hostBuffer.Size());
EXPECT_EQ(ptrInfo.MemFlags.ui32.CoarseGrain, 0);
if (is_dgpu()) {
if (hsakmt_is_dgpu()) {
EXPECT_EQ((HSAuint64)ptrInfo.NMappedNodes, nGPU);
// Check NMappedNodes again after unmapping the memory
hsaKmtUnmapMemoryToGPU(hostBuffer.As<void*>());
@@ -1137,7 +1137,7 @@ TEST_F(KFDMemoryTest, QueryPointerInfo) {
* User pointers registered with SVM API, does not create vm_object_t.
* Therefore, pointer info can not be queried.
*/
if (is_dgpu() && mem != hsaBuffer.As<void*>()) {
if (hsakmt_is_dgpu() && mem != hsaBuffer.As<void*>()) {
EXPECT_SUCCESS(hsaKmtQueryPointerInfo((void *)(&mem[0]), &ptrInfo));
EXPECT_EQ(ptrInfo.Type, HSA_POINTER_REGISTERED_USER);
EXPECT_EQ(ptrInfo.CPUAddress, &mem[0]);
@@ -1164,7 +1164,7 @@ TEST_F(KFDMemoryTest, QueryPointerInfo) {
EXPECT_SUCCESS(hsaKmtQueryPointerInfo(reinterpret_cast<void *>(address), &ptrInfo));
EXPECT_EQ(ptrInfo.Type, HSA_POINTER_ALLOCATED);
EXPECT_EQ(ptrInfo.CPUAddress, hostBuffer.As<void*>());
if (is_dgpu() && &mem[1] != hsaBuffer.As<HSAuint32 *>() + 1) {
if (hsakmt_is_dgpu() && &mem[1] != hsaBuffer.As<HSAuint32 *>() + 1) {
EXPECT_SUCCESS(hsaKmtQueryPointerInfo((void *)(&mem[1]), &ptrInfo));
EXPECT_EQ(ptrInfo.Type, HSA_POINTER_REGISTERED_USER);
EXPECT_EQ(ptrInfo.CPUAddress, &mem[0]);
@@ -1329,7 +1329,7 @@ TEST_F(KFDMemoryTest, PtraceAccess) {
TEST_F(KFDMemoryTest, PtraceAccessInvisibleVram) {
char *hsaDebug = getenv("HSA_DEBUG");
if (!is_dgpu()) {
if (!hsakmt_is_dgpu()) {
LOG() << "Skipping test: There is no VRAM on APU." << std::endl;
return;
}
@@ -1476,7 +1476,7 @@ void CatchSignal(int IntrSignal) {
TEST_F(KFDMemoryTest, SignalHandling) {
TEST_START(TESTPROFILE_RUNALL)
if (!is_dgpu()) {
if (!hsakmt_is_dgpu()) {
LOG() << "Skipping test: Test not supported on APU." << std::endl;
return;
}
projects/rocr-runtime/libhsakmt/tests/kfdtest/src/KFDPerformanceTest.cpp
+2 -2
View File
@@ -139,7 +139,7 @@ testNodeToNodes(HSAuint32 n1, const HSAuint32 *const n2Array, int n, P2PDirectio
TEST_F(KFDPerformanceTest, P2PBandWidthTest) {
TEST_START(TESTPROFILE_RUNALL);
if (!is_dgpu()) {
if (!hsakmt_is_dgpu()) {
LOG() << "Skipping test: Can't have 2 APUs on the same system." << std::endl;
return;
}
@@ -321,7 +321,7 @@ exit:
TEST_F(KFDPerformanceTest, P2POverheadTest) {
TEST_START(TESTPROFILE_RUNALL);
if (!is_dgpu()) {
if (!hsakmt_is_dgpu()) {
LOG() << "Skipping test: Can't have 2 APUs on the same system." << std::endl;
return;
}
projects/rocr-runtime/libhsakmt/tests/kfdtest/src/KFDQMTest.cpp
+2 -2
View File
@@ -399,7 +399,7 @@ TEST_F(KFDQMTest, SdmaConcurrentCopies) {
#define NPACKETS 1
#define COPY_SIZE (BUFFER_SIZE / NPACKETS)
HsaMemoryBuffer srcBuf(BUFFER_SIZE, 0, true);
HsaMemoryBuffer dstBuf(BUFFER_SIZE, defaultGPUNode, false, is_dgpu() ? true : false);
HsaMemoryBuffer dstBuf(BUFFER_SIZE, defaultGPUNode, false, hsakmt_is_dgpu() ? true : false);
SDMAQueue queue;
@@ -1493,7 +1493,7 @@ sdma_fill(HSAint32 node, void *dst, unsigned int data, HSAuint64 size) {
TEST_F(KFDQMTest, P2PTest) {
TEST_START(TESTPROFILE_RUNALL);
if (!is_dgpu()) {
if (!hsakmt_is_dgpu()) {
LOG() << "Skipping test: Two GPUs are required, but no dGPUs are present." << std::endl;
return;
}
projects/rocr-runtime/libhsakmt/tests/kfdtest/src/KFDSVMEvictTest.cpp
+4 -4
View File
@@ -57,7 +57,7 @@ void KFDSVMEvictTest::TearDown() {
}
HSAint32 KFDSVMEvictTest::GetBufferCounter(HSAuint64 vramSize, HSAuint64 vramBufSize) {
HSAuint64 vramBufSizeInPages = vramBufSize >> PAGE_SHIFT;
HSAuint64 vramBufSizeInPages = vramBufSize >> HSAKMT_PAGE_SHIFT;
HSAuint64 sysMemSize = GetSysMemSize();
HSAuint64 size, sizeInPages;
HSAuint32 count;
@@ -79,7 +79,7 @@ HSAint32 KFDSVMEvictTest::GetBufferCounter(HSAuint64 vramSize, HSAuint64 vramBuf
if (!xnack_enable && size > (sysMemSize - (sysMemSize >> 4)))
return 0;
sizeInPages = size >> PAGE_SHIFT;
sizeInPages = size >> HSAKMT_PAGE_SHIFT;
count = sizeInPages / (vramBufSizeInPages * N_PROCESSES);
return count;
@@ -105,10 +105,10 @@ HSAint64 KFDSVMEvictTest::GetBufferSize(HSAuint64 vramSize, HSAuint32 count,
if (!xnack_enable && size > (sysMemSize - (sysMemSize >> 4)))
return 0;
sizeInPages = size >> PAGE_SHIFT;
sizeInPages = size >> HSAKMT_PAGE_SHIFT;
vramBufSizeInPages = sizeInPages / (count * N_PROCESSES);
return vramBufSizeInPages << PAGE_SHIFT;
return vramBufSizeInPages << HSAKMT_PAGE_SHIFT;
}
void KFDSVMEvictTest::AllocBuffers(HSAuint32 defaultGPUNode, HSAuint32 count, HSAuint64 vramBufSize,
projects/rocr-runtime/libhsakmt/tests/kfdtest/src/KFDSVMRangeTest.cpp
+3 -3
View File
@@ -1321,7 +1321,7 @@ TEST_P(KFDSVMRangeTest, ReadOnlyRangeTest) {
int childStatus;
waitpid(pid, &childStatus, 0);
if (is_dgpu()) {
if (hsakmt_is_dgpu()) {
EXPECT_EQ(true, WIFEXITED(childStatus));
EXPECT_EQ(0, WEXITSTATUS(childStatus));
} else {
@@ -1447,8 +1447,8 @@ unsigned int ReadSMIEventThread(void* p) {
EXPECT_EQ(sscanf(msg, "%x %ld -%d @%lx(%d) %d->%x %x:%d %d\n", &event_id, &timestamp, &pid,
&addr, &size, &unused, &unused, &unused, &unused, &trigger), 10);
EXPECT_EQ(event_id, HSA_SMI_EVENT_MIGRATE_START);
EXPECT_EQ((HSAuint64 *)(addr << PAGE_SHIFT), pArgs->pBuf);
EXPECT_EQ(size << PAGE_SHIFT, pArgs->BufSize);
EXPECT_EQ((HSAuint64 *)(addr << HSAKMT_PAGE_SHIFT), pArgs->pBuf);
EXPECT_EQ(size << HSAKMT_PAGE_SHIFT, pArgs->BufSize);
EXPECT_EQ(pid, getpid());
EXPECT_EQ(trigger, HSA_MIGRATE_TRIGGER_PREFETCH);
close(fd);
projects/rocr-runtime/libhsakmt/tests/kfdtest/src/KFDTestUtil.cpp
+7 -7
View File
@@ -130,10 +130,10 @@ HSAKMT_STATUS CreateQueueTypeEvent(
return hsaKmtCreateEvent(&Descriptor, ManualReset, IsSignaled, Event);
}
static bool is_dgpu_dev = false;
static bool hsakmt_is_dgpu_dev = false;
bool is_dgpu() {
return is_dgpu_dev;
bool hsakmt_is_dgpu() {
return hsakmt_is_dgpu_dev;
}
bool hasPciAtomicsSupport(int node) {
@@ -213,9 +213,9 @@ unsigned int FamilyIdFromNode(const HsaNodeProperties *props) {
}
if (props->NumCPUCores && props->NumFComputeCores)
is_dgpu_dev = false;
hsakmt_is_dgpu_dev = false;
else
is_dgpu_dev = true;
hsakmt_is_dgpu_dev = true;
return familyId;
}
@@ -302,7 +302,7 @@ HsaMemoryBuffer::HsaMemoryBuffer(HSAuint64 size, unsigned int node, bool zero, b
EXPECT_EQ(m_Flags.ui32.HostAccess, 1);
EXPECT_SUCCESS(hsaKmtAllocMemory(m_Node, m_Size, m_Flags, &m_pBuf));
if (is_dgpu()) {
if (hsakmt_is_dgpu()) {
if (map_specific_gpu)
EXPECT_SUCCESS(hsaKmtMapMemoryToGPUNodes(m_pBuf, m_Size, NULL, mapFlags, 1, &m_Node));
else
@@ -531,7 +531,7 @@ HsaMemoryBuffer::~HsaMemoryBuffer() {
hsaKmtUnmapMemoryToGPU(m_pUser);
hsaKmtDeregisterMemory(m_pUser);
} else if (m_pBuf != NULL) {
if (is_dgpu()) {
if (hsakmt_is_dgpu()) {
if (m_MappedNodes) {
hsaKmtUnmapMemoryToGPU(m_pBuf);
}
projects/rocr-runtime/libhsakmt/tests/kfdtest/src/KFDTestUtil.hpp
+2 -2
View File
@@ -33,7 +33,7 @@
class BaseQueue;
#define ARRAY_SIZE(_x) (sizeof(_x)/sizeof(_x[0]))
#define ALIGN_UP(x, align) (((uint64_t)(x) + (align) - 1) & ~(uint64_t)((align)-1))
#define CounterToNanoSec(x) ((x) * 1000 / (is_dgpu() ? 27 : 100))
#define CounterToNanoSec(x) ((x) * 1000 / (hsakmt_is_dgpu() ? 27 : 100))
void WaitUntilInput();
HSAKMT_STATUS fscanf_dec(const char *file, uint32_t *num);
@@ -50,7 +50,7 @@ bool GetHwCapabilityHWS();
HSAKMT_STATUS CreateQueueTypeEvent(bool ManualReset, bool IsSignaled, unsigned int NodeId, HsaEvent** Event);
bool is_dgpu();
bool hsakmt_is_dgpu();
bool isTonga(const HsaNodeProperties *props);
bool hasPciAtomicsSupport(int node);
unsigned int FamilyIdFromNode(const HsaNodeProperties *props);
projects/rocr-runtime/libhsakmt/tests/kfdtest/src/KFDTopologyTest.cpp
+1 -1
View File
@@ -126,7 +126,7 @@ TEST_F(KFDTopologyTest, GpuvmApertureValidate) {
for (unsigned i = 0; i < GpuNodes.size(); i++) {
pNodeProperties = m_NodeInfo.GetNodeProperties(GpuNodes.at(i));
if (pNodeProperties != NULL) {
if (!is_dgpu() && !(FamilyIdFromNode(pNodeProperties) == FAMILY_KV)) {
if (!hsakmt_is_dgpu() && !(FamilyIdFromNode(pNodeProperties) == FAMILY_KV)) {
LOG() << "Skipping test: GPUVM framebuffer heap not exposed on APU except Kaveri." << std::endl;
return;
}
projects/rocr-runtime/libhsakmt/tests/kfdtest/src/OSWrapper.hpp
+1 -1
View File
@@ -33,7 +33,7 @@
#ifndef PAGE_SIZE
#define PAGE_SIZE (1<<12)
#define PAGE_SHIFT (12)
#define HSAKMT_PAGE_SHIFT (12)
#endif
enum TEXTCOLOR {
projects/rocr-runtime/libhsakmt/tests/kfdtest/src/PM4Packet.cpp
+2 -2
View File
@@ -55,7 +55,7 @@ void PM4WriteDataPacket::InitPacket(unsigned int *destBuf, void *data) {
m_pPacketData->bitfields2.dst_sel = dst_sel_mec_write_data_MEMORY_5; // memory-async
m_pPacketData->bitfields2.addr_incr = addr_incr_mec_write_data_INCREMENT_ADDR_0; // increment addr
m_pPacketData->bitfields2.wr_confirm = wr_confirm_mec_write_data_WAIT_FOR_CONFIRMATION_1;
m_pPacketData->bitfields2.atc = is_dgpu() ?
m_pPacketData->bitfields2.atc = hsakmt_is_dgpu() ?
atc_write_data_NOT_USE_ATC_0 : atc_write_data_USE_ATC_1;
m_pPacketData->bitfields2.cache_policy = cache_policy_mec_write_data_BYPASS_2;
@@ -97,7 +97,7 @@ void PM4ReleaseMemoryPacket::InitPacketCI(bool isPolling, uint64_t address,
pkt->bitfields2.l2_wb = 1;
pkt->bitfields2.l2_inv = 1;
pkt->bitfields2.cache_policy = cache_policy_mec_release_mem_BYPASS_2;
pkt->bitfields2.atc = is_dgpu() ?
pkt->bitfields2.atc = hsakmt_is_dgpu() ?
atc_mec_release_mem_ci_NOT_USE_ATC_0 :
atc_mec_release_mem_ci_USE_ATC_1; // ATC setting for fences and timestamps to the MC or TCL2.
pkt->bitfields3.dst_sel = dst_sel_mec_release_mem_MEMORY_CONTROLLER_0;
projects/rocr-runtime/libhsakmt/tests/rdma/simple/drv/amdp2ptest.c
+2 -2
View File
@@ -304,7 +304,7 @@ static int amdp2ptest_mmap(struct file *filp, struct vm_area_struct *vma)
struct va_pages_node *va_pages = NULL;
struct amdp2ptest_pages_list *list = filp->private_data;
struct list_head *p, *n;
uint64_t gpu_va = vma->vm_pgoff << PAGE_SHIFT;
uint64_t gpu_va = vma->vm_pgoff << HSAKMT_PAGE_SHIFT;
MSG_INFO("Mapping to CPU user space\n");
MSG_INFO("Begin vm_start 0x%lx, vm_end 0x%lx\n", vma->vm_start, vma->vm_end);
@@ -340,7 +340,7 @@ static int amdp2ptest_mmap(struct file *filp, struct vm_area_struct *vma)
addr, sg->dma_address, size);
ret = remap_pfn_range(vma,
addr,
sg->dma_address >> PAGE_SHIFT,
sg->dma_address >> HSAKMT_PAGE_SHIFT,
size,
vma->vm_page_prot);
if (ret) {