diff --git a/include/hsakmt.h b/include/hsakmt.h index ed5aff7144..9084079882 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -330,6 +330,13 @@ hsaKmtSetQueueCUMask( HSAuint32* QueueCUMask //IN ); +HSAKMT_STATUS +HSAKMTAPI +hsaKmtGetQueueInfo( + HSA_QUEUEID QueueId, //IN + HsaQueueInfo *QueueInfo //IN +); + /** Allows an HSA process to set/change the default and alternate memory coherency, before starting to dispatch. */ diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index df83817010..eb640d7014 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -582,6 +582,25 @@ typedef enum _HSA_QUEUE_TYPE HSA_QUEUE_TYPE_SIZE = 0xFFFFFFFF //aligns to 32bit enum } HSA_QUEUE_TYPE; +typedef struct +{ + HSAuint32 QueueDetailError; // HW specific queue error state + HSAuint32 QueueTypeExtended; // HW specific queue type info. + // 0 = no information + HSAuint32 NumCUAssigned; // size of *CUMaskInfo bit array, Multiple + // of 32, 0 = no information + HSAuint32* CUMaskInfo; // runtime/system CU assignment for realtime + // queue & reserved CU priority. Ptr to + // bit-array, each bit represents one CU. + // NULL = no information + HSAuint32* UserContextSaveArea; // reference to user space context save area + HSAuint64 SaveAreaSizeInBytes; // Must be 4-Byte aligned + HSAuint32* ControlStackTop; // ptr to the TOS + HSAuint64 ControlStackUsedInBytes; // Must be 4-Byte aligned + HSAuint64 Reserved1; // runtime/system CU assignment + HSAuint64 Reserved2; // runtime/system CU assignment +} HsaQueueInfo; + typedef struct _HsaQueueResource { HSA_QUEUEID QueueId; /** queue ID */ diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 8f833bdeb1..1ff1271e91 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -82,6 +82,14 @@ struct kfd_ioctl_set_cu_mask_args { uint64_t cu_mask_ptr; /* to KFD */ }; +struct kfd_ioctl_get_queue_wave_state_args { + uint64_t ctl_stack_address; /* to KFD */ + uint32_t ctl_stack_used_size; /* from KFD */ + uint32_t save_area_used_size; /* from KFD */ + uint32_t queue_id; /* to KFD */ + uint32_t pad; +}; + /* For kfd_ioctl_set_memory_policy_args.default_policy and alternate_policy */ #define KFD_IOC_CACHE_POLICY_COHERENT 0 #define KFD_IOC_CACHE_POLICY_NONCOHERENT 1 @@ -526,11 +534,14 @@ struct kfd_ioctl_cross_memory_copy_args { #define AMDKFD_IOC_CROSS_MEMORY_COPY \ AMDKFD_IOWR(0x1F, struct kfd_ioctl_cross_memory_copy_args) +#define AMDKFD_IOC_GET_QUEUE_WAVE_STATE \ + AMDKFD_IOWR(0x20, struct kfd_ioctl_get_queue_wave_state_args) + /* TODO: remove this */ #define AMDKFD_IOC_OPEN_GRAPHIC_HANDLE \ - AMDKFD_IOWR(0x20, struct kfd_ioctl_open_graphic_handle_args) + AMDKFD_IOWR(0x21, struct kfd_ioctl_open_graphic_handle_args) #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x21 +#define AMDKFD_COMMAND_END 0x22 #endif diff --git a/src/libhsakmt.h b/src/libhsakmt.h index be57fc59f1..0cc45ea200 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -59,6 +59,7 @@ extern int PAGE_SHIFT; do { if ((uint64_t)PORT_VPTR_TO_UINT64(x) % PAGE_SIZE) return HSAKMT_STATUS_INVALID_PARAMETER; } while(0) #define ALIGN_UP(x,align) (((uint64_t)(x) + (align) - 1) & ~(uint64_t)((align)-1)) +#define ALIGN_UP_32(x,align) (((uint32_t)(x) + (align) - 1) & ~(uint32_t)((align)-1)) #define PAGE_ALIGN_UP(x) ALIGN_UP(x,PAGE_SIZE) #define BITMASK(n) (((n) < sizeof(1ULL) * CHAR_BIT ? (1ULL << (n)) : 0) - 1ULL) #define ARRAY_LEN(array) (sizeof(array) / sizeof(array[0])) diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index e955a31c97..436e33bf8c 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -55,6 +55,7 @@ hsaKmtSetTrapHandler; hsaKmtGetTileConfig; hsaKmtQueryPointerInfo; hsaKmtSetMemoryUserData; +hsaKmtGetQueueInfo; local: *; }; diff --git a/src/queues.c b/src/queues.c index 7fbd1083c7..9860a6d6ec 100644 --- a/src/queues.c +++ b/src/queues.c @@ -122,7 +122,14 @@ struct queue { void *ctx_save_restore; uint32_t ctx_save_restore_size; uint32_t ctl_stack_size; + void *ctl_stack_copy; const struct device_info *dev_info; + /* This queue structure is allocated from GPU with page aligned size + * but only small bytes are used. We use the extra space in the end for + * cu_mask bits array. + */ + uint32_t cu_mask_count; /* in bits */ + uint32_t cu_mask[0]; }; struct process_doorbells { @@ -416,6 +423,11 @@ static void free_queue(struct queue *q) free_exec_aligned_memory(q->ctx_save_restore, q->ctx_save_restore_size, PAGE_SIZE, q->dev_info->asic_family); + if (q->ctl_stack_copy) + free_exec_aligned_memory(q->ctl_stack_copy, + q->ctl_stack_size, + PAGE_SIZE, q->dev_info->asic_family); + free_exec_aligned_memory((void *)q, sizeof(*q), PAGE_SIZE, q->dev_info->asic_family); } @@ -451,6 +463,11 @@ static int handle_concrete_asic(struct queue *q, return HSAKMT_STATUS_NO_MEMORY; args->ctx_save_restore_address = (uintptr_t)q->ctx_save_restore; + if (IS_SOC15(q->dev_info->asic_family)) { + q->ctl_stack_copy = malloc(q->ctl_stack_size); + if (q->ctl_stack_copy == NULL) + return HSAKMT_STATUS_NO_MEMORY; + } } } @@ -479,6 +496,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, unsigned int doorbell_offset; struct device_info *dev_info; int err; + HsaNodeProperties props; + uint32_t cu_num, i; CHECK_KFD_OPEN(); @@ -501,6 +520,19 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, memset(q, 0, sizeof(*q)); + /* By default, CUs are all turned on. Initialize cu_mask to '1 + * for all CU bits. + */ + if (hsaKmtGetNodeProperties(NodeId, &props)) + q->cu_mask_count = 0; + else { + cu_num = props.NumFComputeCores / props.NumSIMDPerCU; + /* cu_mask_count counts bits. It must be multiple of 32 */ + q->cu_mask_count = ALIGN_UP_32(cu_num, 32); + for (i = 0; i < cu_num; i++) + q->cu_mask[i/32] |= (1 << (i % 32)); + } + struct kfd_ioctl_create_queue_args args; memset(&args, 0, sizeof(args)); @@ -534,7 +566,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, return err; } - args.read_pointer_address = QueueResource->QueueRptrValue; args.write_pointer_address = QueueResource->QueueWptrValue; args.ring_base_address = (uintptr_t)QueueAddress; @@ -662,6 +693,64 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetQueueCUMask(HSA_QUEUEID QueueId, if (err == -1) return HSAKMT_STATUS_ERROR; + memcpy(q->cu_mask, QueueCUMask, CUMaskCount / 8); + q->cu_mask_count = CUMaskCount; + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtGetQueueInfo( + HSA_QUEUEID QueueId, + HsaQueueInfo *QueueInfo +) +{ + struct queue *q = PORT_UINT64_TO_VPTR(QueueId); + struct kfd_ioctl_get_queue_wave_state_args args; + uintptr_t ctl_stack_base_addr = 0; + + CHECK_KFD_OPEN(); + + if (QueueInfo == NULL || q == NULL) + return HSAKMT_STATUS_INVALID_PARAMETER; + + if (q->ctx_save_restore == NULL) + return HSAKMT_STATUS_ERROR; + + memset(&args, 0, sizeof(args)); + args.queue_id = q->queue_id; + + if (IS_SOC15(q->dev_info->asic_family)) { + /* From SOC15 onwards the control stack is kept in the MQD. + * The save area is at the beginning of the allocated space. + * Request a copy of the control stack from the KFD. + */ + ctl_stack_base_addr = (uintptr_t)q->ctl_stack_copy; + QueueInfo->UserContextSaveArea = q->ctx_save_restore; + args.ctl_stack_address = ctl_stack_base_addr; + } else { + /* Before SOC15 the control stack is already in userspace. + * The save area immediately follows the control stack. + */ + ctl_stack_base_addr = (uintptr_t)q->ctx_save_restore; + QueueInfo->UserContextSaveArea = (void *) + ((uintptr_t)q->ctx_save_restore + q->ctl_stack_size); + args.ctl_stack_address = 0; + } + + if (kmtIoctl(kfd_fd, AMDKFD_IOC_GET_QUEUE_WAVE_STATE, &args) < 0) + return HSAKMT_STATUS_ERROR; + + QueueInfo->ControlStackTop = (void *)(ctl_stack_base_addr + + q->ctl_stack_size - args.ctl_stack_used_size); + QueueInfo->SaveAreaSizeInBytes = args.save_area_used_size; + QueueInfo->ControlStackUsedInBytes = args.ctl_stack_used_size; + QueueInfo->NumCUAssigned = q->cu_mask_count; + QueueInfo->CUMaskInfo = q->cu_mask; + QueueInfo->QueueDetailError = 0; + QueueInfo->QueueTypeExtended = 0; + return HSAKMT_STATUS_SUCCESS; }