From e2c353dc0dd73853b7e5bac8838ce7080cf248d7 Mon Sep 17 00:00:00 2001 From: Jay Cornwall Date: Thu, 22 Feb 2018 13:06:15 -0600 Subject: [PATCH] Allocate EOP queue local to GPU On discrete GPUs place the EOP queue in VRAM. The reader/writer of this queue is the CP and the size is small. Dispatch latency improves through lower read latency in AQL completion phase. Change-Id: Id8351dcddbd21fd7c7d699803c96434c9132db71 Signed-off-by: Jay Cornwall --- src/events.c | 2 +- src/libhsakmt.h | 3 ++- src/queues.c | 18 ++++++++++-------- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/events.c b/src/events.c index 01f4db2992..65330c90ee 100644 --- a/src/events.c +++ b/src/events.c @@ -76,7 +76,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateEvent(HsaEventDescriptor *EventDesc, if (is_dgpu && !events_page) { events_page = allocate_exec_aligned_memory_gpu( - KFD_SIGNAL_EVENT_LIMIT * 8, PAGE_SIZE, 0, true); + KFD_SIGNAL_EVENT_LIMIT * 8, PAGE_SIZE, 0, true, false); if (!events_page) { pthread_mutex_unlock(&hsakmt_mutex); return HSAKMT_STATUS_ERROR; diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 07402ffef5..48435c477b 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -123,7 +123,8 @@ HSAKMT_STATUS topology_get_asic_family(uint16_t device_id, HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags); void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, - uint32_t NodeId, bool NonPaged); + uint32_t NodeId, bool NonPaged, + bool DeviceLocal); void free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align); HSAKMT_STATUS init_process_doorbells(unsigned int NumNodes); void destroy_process_doorbells(void); diff --git a/src/queues.c b/src/queues.c index 8f035a12d4..613180cedb 100644 --- a/src/queues.c +++ b/src/queues.c @@ -350,7 +350,8 @@ static bool update_ctx_save_restore_size(uint32_t nodeid, struct queue *q) } void *allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, - uint32_t NodeId, bool nonPaged) + uint32_t NodeId, bool nonPaged, + bool DeviceLocal) { void *mem; HSAuint64 gpu_va; @@ -358,14 +359,14 @@ void *allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, HSAKMT_STATUS ret; flags.Value = 0; - flags.ui32.HostAccess = 1; + flags.ui32.HostAccess = !DeviceLocal; flags.ui32.ExecuteAccess = 1; flags.ui32.NonPaged = nonPaged; flags.ui32.PageSize = HSA_PAGE_SIZE_4KB; size = ALIGN_UP(size, align); - ret = hsaKmtAllocMemory(0, size, flags, &mem); + ret = hsaKmtAllocMemory(DeviceLocal ? NodeId : 0, size, flags, &mem); if (ret != HSAKMT_STATUS_SUCCESS) return NULL; @@ -399,11 +400,12 @@ void free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align) */ static void *allocate_exec_aligned_memory(uint32_t size, enum asic_family_type type, - uint32_t NodeId) + uint32_t NodeId, + bool DeviceLocal) { if (IS_DGPU(type)) return allocate_exec_aligned_memory_gpu(size, PAGE_SIZE, NodeId, - false); + DeviceLocal, DeviceLocal); return allocate_exec_aligned_memory_cpu(size); } @@ -441,7 +443,7 @@ static int handle_concrete_asic(struct queue *q, q->eop_buffer = allocate_exec_aligned_memory(q->dev_info->eop_buffer_size, dev_info->asic_family, - NodeId); + NodeId, true); if (!q->eop_buffer) return HSAKMT_STATUS_NO_MEMORY; @@ -455,7 +457,7 @@ static int handle_concrete_asic(struct queue *q, q->ctx_save_restore = allocate_exec_aligned_memory(q->ctx_save_restore_size, dev_info->asic_family, - NodeId); + NodeId, false); if (!q->ctx_save_restore) return HSAKMT_STATUS_NO_MEMORY; @@ -506,7 +508,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, struct queue *q = allocate_exec_aligned_memory(sizeof(*q), dev_info->asic_family, - NodeId); + NodeId, false); if (!q) return HSAKMT_STATUS_NO_MEMORY;