From 0edaa45b8ab954c9f416a73444acbabebd224fc8 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Thu, 30 Jun 2022 10:19:25 -0400 Subject: [PATCH] Only allow pairwise CU enable for devices with WGPs A work group processor (WGP) require both its CU to be enabled in order to be enabled. The KFD will round robin distribute by even-indexed pairs so enforce this requirement for runtime set mask calls. Change-Id: Ic46661b01f398aa1fe24d96b5c9c31f122f967a3 [ROCm/ROCR-Runtime commit: f60068753769a4b650314e9d1f2370cd21f50a51] --- .../runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp | 9 +++++++++ .../rocr-runtime/runtime/hsa-runtime/inc/hsa_ext_amd.h | 3 +++ 2 files changed, 12 insertions(+) diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp index 55feab0c29..e8febcfee6 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp @@ -1116,6 +1116,15 @@ hsa_status_t AqlQueue::SetCUMasking(uint32_t num_cu_mask_count, const uint32_t* // Apply mask if non-default or not queue initialization. ScopedAcquire lock(&mask_lock_); if ((!cu_mask_.empty()) || (num_cu_mask_count != 0) || (!global_mask.empty())) { + + // Devices with WGPs must conform to even-indexed contiguous pairwise CU enablement. + if (agent_->isa()->GetMajorVersion() >= 10) { + for (int i = 0; i < mask.size() * 32; i += 2) { + uint32_t cu_pair = (mask[i / 32] >> (i % 32)) & 0x3; + if (cu_pair && cu_pair != 0x3) return HSA_STATUS_ERROR_INVALID_ARGUMENT; + } + } + HSAKMT_STATUS ret = hsaKmtSetQueueCUMask(queue_id_, mask.size() * 32, reinterpret_cast(&mask[0])); if (ret != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; diff --git a/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_ext_amd.h b/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_ext_amd.h index 8ce46ac6df..290c9ac845 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_ext_amd.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_ext_amd.h @@ -833,6 +833,9 @@ hsa_status_t HSA_API hsa_amd_image_get_info_max_dim(hsa_agent_t agent, * * @retval ::HSA_STATUS_ERROR_INVALID_ARGUMENT @p num_cu_mask_count is not * a multiple of 32 or @p num_cu_mask_count is not 0 and cu_mask is NULL. + * Devices with work group processors must even-index contiguous pairwise + * CU enable e.g. 0x33(b'110011) is valid while 0x5(0x101) and 0x6(b'0110) + * are invalid. * */ hsa_status_t HSA_API hsa_amd_queue_cu_set_mask(const hsa_queue_t* queue,