From 0a2ea9ef55403341a43df4c96dd946c4641a7f79 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Tue, 16 Dec 2025 07:26:14 -0500 Subject: [PATCH] hsakmt: Expose and use CWSR and Control stack sizes (#2200) * hsakmt: Expose CWSR and Control stack sizes This is better than hardcoding values and hoping that they align with KFD's definitions Signed-off-by: Kent Russell * hsakmt: Use CwsrSize and CtlStackSize if available If KFD is providing the CwsrSize and CtlStackSize, use the maximum of those and the old calculations for the ctx_save_restore_size and ctl_stack_size defined in the queue Signed-off-by: Kent Russell * hsakmt: Add warning when ABI<1.20 on GFX1151 CwsrSize and CtlStackSize are reported by KFD ABI 1.20. GFX1151 specifically may have some issues if these regions are misaligned, so report a strong warning during topology initialization if the system is GFX1151 but is using KFD ABI < 1.20 Signed-off-by: Kent Russell --------- Signed-off-by: Kent Russell --- .../libhsakmt/include/hsakmt/hsakmttypes.h | 2 ++ projects/rocr-runtime/libhsakmt/src/queues.c | 9 +++++++-- projects/rocr-runtime/libhsakmt/src/topology.c | 11 +++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/projects/rocr-runtime/libhsakmt/include/hsakmt/hsakmttypes.h b/projects/rocr-runtime/libhsakmt/include/hsakmt/hsakmttypes.h index 53ea25a714..5516d6f7d9 100644 --- a/projects/rocr-runtime/libhsakmt/include/hsakmt/hsakmttypes.h +++ b/projects/rocr-runtime/libhsakmt/include/hsakmt/hsakmttypes.h @@ -346,6 +346,8 @@ typedef struct _HsaNodeProperties HSAuint32 SGPRSizePerCU; // SGPR size in bytes per CU HSAuint32 NumXcc; // Number of XCC + HSAuint32 CwsrSize; // Size of the CWSR + HSAuint32 CtlStackSize; // Size of the control stack HSAuint32 KFDGpuID; // GPU Hash ID generated by KFD HSAuint32 FamilyID; // GPU family id diff --git a/projects/rocr-runtime/libhsakmt/src/queues.c b/projects/rocr-runtime/libhsakmt/src/queues.c index c2a00734ea..5a92bdbc48 100644 --- a/projects/rocr-runtime/libhsakmt/src/queues.c +++ b/projects/rocr-runtime/libhsakmt/src/queues.c @@ -342,8 +342,13 @@ static bool update_ctx_save_restore_size(HsaKFDContext *ctx, uint32_t nodeid, st q->debug_memory_size = ALIGN_UP(wave_num * DEBUGGER_BYTES_PER_WAVE, DEBUGGER_BYTES_ALIGN); - q->ctx_save_restore_size = q->ctl_stack_size - + PAGE_ALIGN_UP(wg_data_size); + /* Keep calculating it in case we are using an older kernel, but if we have + * the CtlStackSize and CwsrSize from KFD, use that as the definitive value + */ + q->ctx_save_restore_size = node.CwsrSize > 0 ? node.CwsrSize : + q->ctl_stack_size + PAGE_ALIGN_UP(wg_data_size); + q->ctl_stack_size = node.CtlStackSize > 0 ? node.CtlStackSize : q->ctl_stack_size; + return true; } return false; diff --git a/projects/rocr-runtime/libhsakmt/src/topology.c b/projects/rocr-runtime/libhsakmt/src/topology.c index 6db13f9073..8786000e79 100644 --- a/projects/rocr-runtime/libhsakmt/src/topology.c +++ b/projects/rocr-runtime/libhsakmt/src/topology.c @@ -1212,6 +1212,10 @@ static HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, props->NumSdmaQueuesPerEngine = prop_val; else if (strcmp(prop_name, "num_cp_queues") == 0) props->NumCpQueues = prop_val; + else if (strcmp(prop_name, "cwsr_size") == 0) + props->CwsrSize = prop_val; + else if (strcmp(prop_name, "ctl_stack_size") == 0) + props->CtlStackSize = prop_val; else if (strcmp(prop_name, "num_xcc") == 0) props->NumXcc = prop_val; else if (strcmp(prop_name, "family_id") == 0) @@ -2221,6 +2225,13 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAcquireSystemPropertiesCtx(HsaKFDContext *ctx, *SystemProperties = *g_system; + for (int node = 0; node < g_system->NumNodes; node++) { + if (hsakmt_get_gfxv_by_node_id(node) == GFX_VERSION_GFX1151 && + hsakmt_kfd_version_info.KernelInterfaceMajorVersion == 1 && + hsakmt_kfd_version_info.KernelInterfaceMinorVersion < 20) + pr_err_once("WARNING: KFD ABI 1.20+ is recommended for gfx1151. Current KFD ABI is %i.%i. This may result in faults, crashes and other application instability\n", hsakmt_kfd_version_info.KernelInterfaceMajorVersion, hsakmt_kfd_version_info.KernelInterfaceMinorVersion); + } + goto out; init_doorbells_failed: