commit 79a63cf292c8f674b9fbfed108b94c1d239ce62b Author: Flora Cui Date: Tue Apr 30 09:01:09 2024 -0500 wsl/hsakmt: initial commit Signed-off-by: lyndonli Signed-off-by: Horatio Zhang Signed-off-by: Shi.Leslie Signed-off-by: LonglongYao Signed-off-by: tiancyin Signed-off-by: Frank Min Signed-off-by: Aaron Liu Signed-off-by: Shane Xiao Signed-off-by: Lang Yu Signed-off-by: Feifei Xu Signed-off-by: Ruili Ji Signed-off-by: Qiang Yu Signed-off-by: Flora Cui diff --git a/debug.cpp b/debug.cpp new file mode 100644 index 0000000000..5152aaf483 --- /dev/null +++ b/debug.cpp @@ -0,0 +1,118 @@ +/* + * Copyright © 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include + +#include "libhsakmt.h" + +static uint32_t runtime_capabilities_mask = 0; + +HSAKMT_STATUS HSAKMTAPI hsaKmtDbgRegister(HSAuint32 NodeId) { + CHECK_DXG_OPEN(); + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtDbgUnregister(HSAuint32 NodeId) { + CHECK_DXG_OPEN(); + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtDbgWavefrontControl( + HSAuint32 NodeId, HSA_DBG_WAVEOP Operand, HSA_DBG_WAVEMODE Mode, + HSAuint32 TrapId, HsaDbgWaveMessage *DbgWaveMsgRing) { + CHECK_DXG_OPEN(); + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtDbgAddressWatch( + HSAuint32 NodeId, HSAuint32 NumWatchPoints, HSA_DBG_WATCH_MODE WatchMode[], + void *WatchAddress[], HSAuint64 WatchMask[], HsaEvent *WatchEvent[]) { + CHECK_DXG_OPEN(); + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtCheckRuntimeDebugSupport(void) { + CHECK_DXG_OPEN(); + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtRuntimeEnable(void *rDebug, bool setupTtmp) { + HSAKMT_STATUS result = hsaKmtCheckRuntimeDebugSupport(); + + if (result) + return result; + + assert(false); + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtRuntimeDisable(void) { + HSAKMT_STATUS result = hsaKmtCheckRuntimeDebugSupport(); + + if (result) + return result; + + assert(false); + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtGetRuntimeCapabilities(HSAuint32 *caps_mask) { + CHECK_DXG_OPEN(); + *caps_mask = runtime_capabilities_mask; + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtDbgEnable(void **runtime_info, + HSAuint32 *data_size) { + CHECK_DXG_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} +HSAKMT_STATUS HSAKMTAPI hsaKmtDbgDisable(void) { + CHECK_DXG_OPEN(); + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtDbgGetDeviceData(void **data, + HSAuint32 *n_entries, + HSAuint32 *entry_size) { + CHECK_DXG_OPEN(); + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtDbgGetQueueData(void **data, HSAuint32 *n_entries, + HSAuint32 *entry_size, + bool suspend_queues) { + CHECK_DXG_OPEN(); + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS HSAKMTAPI +hsaKmtDebugTrapIoctl(struct kfd_ioctl_dbg_trap_args *args, HSA_QUEUEID *Queues, + HSAuint64 *DebugReturn) { + CHECK_DXG_OPEN(); + return HSAKMT_STATUS_NOT_SUPPORTED; +} diff --git a/events.cpp b/events.cpp new file mode 100644 index 0000000000..e28491f31c --- /dev/null +++ b/events.cpp @@ -0,0 +1,129 @@ +/* + * Copyright © 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include "libhsakmt.h" + +HSAKMT_STATUS HSAKMTAPI hsaKmtCreateEvent(HsaEventDescriptor *EventDesc, + bool ManualReset, bool IsSignaled, + HsaEvent **Event) { + CHECK_DXG_OPEN(); + + assert(false); + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtDestroyEvent(HsaEvent *Event) { + CHECK_DXG_OPEN(); + + if (!Event) + return HSAKMT_STATUS_INVALID_HANDLE; + + assert(false); + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtSetEvent(HsaEvent *Event) { + CHECK_DXG_OPEN(); + + if (!Event) + return HSAKMT_STATUS_INVALID_HANDLE; + + assert(false); + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtResetEvent(HsaEvent *Event) { + CHECK_DXG_OPEN(); + + if (!Event) + return HSAKMT_STATUS_INVALID_HANDLE; + + assert(false); + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtQueryEventState(HsaEvent *Event) { + CHECK_DXG_OPEN(); + + if (!Event) + return HSAKMT_STATUS_INVALID_HANDLE; + + assert(false); + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnEvent(HsaEvent *Event, + HSAuint32 Milliseconds) { + return hsaKmtWaitOnEvent_Ext(Event, Milliseconds, NULL); +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnEvent_Ext(HsaEvent *Event, + HSAuint32 Milliseconds, + uint64_t *event_age) { + if (!Event) + return HSAKMT_STATUS_INVALID_HANDLE; + + return hsaKmtWaitOnMultipleEvents_Ext(&Event, 1, true, Milliseconds, + event_age); +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnMultipleEvents(HsaEvent *Events[], + HSAuint32 NumEvents, + bool WaitOnAll, + HSAuint32 Milliseconds) { + return hsaKmtWaitOnMultipleEvents_Ext(Events, NumEvents, WaitOnAll, + Milliseconds, NULL); +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnMultipleEvents_Ext(HsaEvent *Events[], + HSAuint32 NumEvents, + bool WaitOnAll, + HSAuint32 Milliseconds, + uint64_t *event_age) { + CHECK_DXG_OPEN(); + + if (!Events) + return HSAKMT_STATUS_INVALID_HANDLE; + + if (NumEvents == 1 && Events[0] == nullptr) { + std::this_thread::sleep_for(std::chrono::microseconds(20)); + return HSAKMT_STATUS_SUCCESS; + } + + assert(false); + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtOpenSMI(HSAuint32 NodeId, int *fd) { + CHECK_DXG_OPEN(); + + pr_debug("[%s] node %d\n", __func__, NodeId); + assert(false); + return HSAKMT_STATUS_SUCCESS; +} diff --git a/globals.cpp b/globals.cpp new file mode 100644 index 0000000000..bb635fef4b --- /dev/null +++ b/globals.cpp @@ -0,0 +1,47 @@ +/* + * Copyright © 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "libhsakmt.h" + +// HSAKMT global data + +int kfd_fd = -1; +unsigned long dxg_open_count; +unsigned long system_properties_count; +pthread_mutex_t hsakmt_mutex = PTHREAD_MUTEX_INITIALIZER; +bool is_dgpu; + +#ifndef PAGE_SIZE +int PAGE_SIZE; +#endif + +int PAGE_SHIFT; + +/* whether to check all dGPUs in the topology support SVM API */ +bool is_svm_api_supported; +/* zfb is mainly used during emulation */ +int zfb_support; +/* handle vendor specific packet */ +int vendor_packet_support; \ No newline at end of file diff --git a/inc/pm4_cmds.h b/inc/pm4_cmds.h new file mode 100644 index 0000000000..82edddc8dd --- /dev/null +++ b/inc/pm4_cmds.h @@ -0,0 +1,1064 @@ +#ifndef _ROCR_PM4_CMDS_H_ +#define _ROCR_PM4_CMDS_H_ + +#include + +#define mmCOMPUTE_NUM_THREAD_X 0x2E07 +#define mmCOMPUTE_PGM_LO 0x2E0C +#define mmCOMPUTE_DISPATCH_SCRATCH_BASE_LO 0x2E10 +#define mmCOMPUTE_PGM_RSRC1 0x2E12 +#define mmCOMPUTE_PGM_RSRC3 0x2E28 +#define mmCOMPUTE_RESOURCE_LIMITS 0x2E15 +#define mmCOMPUTE_USER_DATA_0 0x2E40 + +#define PM4_TYPE_SHIFT 30 +#define PM4_COUNT_SHIFT 16 +#define PM4_OPCODE_SHIFT 8 +#define PM4_SHADER_TYPE_SHIFT 1 + +#define PM4_GFX_SHADER 0 +#define PM4_COMPUTE_SHADER 1 + +#define PM4_TYPE3_HDR(_opc_, _count_) \ + (uint32_t)((3) << PM4_TYPE_SHIFT | \ + ((_count_) - 2) << PM4_COUNT_SHIFT | \ + (_opc_) << PM4_OPCODE_SHIFT) | \ + (PM4_COMPUTE_SHADER << PM4_SHADER_TYPE_SHIFT) + +union PM4_MEC_TYPE_3_HEADER { + struct { + uint32_t reserved1 : 8; ///< reserved + uint32_t opcode : 8; ///< IT opcode + uint32_t count : 14;///< number of DWORDs - 1 in the information body. + uint32_t type : 2; ///< packet identifier. It should be 3 for type 3 packets + }; + uint32_t u32All; +}; + +#define IT_DISPATCH_DIRECT 0x15 +#define IT_ATOMIC_MEM 0x1E +#define IT_INDIRECT_BUFFER 0x3F +#define IT_COPY_DATA 0x40 +#define IT_EVENT_WRITE 0x46 +#define IT_RELEASE_MEM 0x49 +#define IT_ACQUIRE_MEM 0x58 +#define IT_SET_SH_REG 0x76 + +struct PM4_MEC_SET_SH_REG { + union { + PM4_MEC_TYPE_3_HEADER header; + uint32_t ordinal1; + }; + union { + struct { + uint32_t reg_offset:16; + uint32_t reserved1:16; + } bitfields2; + uint32_t ordinal2; + }; +}; + +struct PM4_MEC_DISPATCH_DIRECT { + union { + PM4_MEC_TYPE_3_HEADER header; + uint32_t ordinal1; + }; + uint32_t dim_x; + uint32_t dim_y; + uint32_t dim_z; + uint32_t dispatch_initiator; +}; + +// ------------------------------- MEC_EVENT_WRITE_event_index_enum ------------------------------- +enum MEC_EVENT_WRITE_event_index_enum { + event_index__mec_event_write__other = 0, + event_index__mec_event_write__sample_pipelinestat = 2, + event_index__mec_event_write__cs_partial_flush = 4, + event_index__mec_event_write__sample_streamoutstats__GFX11 = 8, + event_index__mec_event_write__sample_streamoutstats1__GFX11 = 9, + event_index__mec_event_write__sample_streamoutstats2__GFX11 = 10, + event_index__mec_event_write__sample_streamoutstats3__GFX11 = 11, +}; + +enum VGT_EVENT_TYPE { + Reserved_0x00 = 0x00000000, + SAMPLE_STREAMOUTSTATS1 = 0x00000001, + SAMPLE_STREAMOUTSTATS2 = 0x00000002, + SAMPLE_STREAMOUTSTATS3 = 0x00000003, + CACHE_FLUSH_TS = 0x00000004, + CONTEXT_DONE = 0x00000005, + CACHE_FLUSH = 0x00000006, + CS_PARTIAL_FLUSH = 0x00000007, + VGT_STREAMOUT_SYNC = 0x00000008, + VGT_STREAMOUT_RESET = 0x0000000a, + END_OF_PIPE_INCR_DE = 0x0000000b, + END_OF_PIPE_IB_END = 0x0000000c, + RST_PIX_CNT = 0x0000000d, + BREAK_BATCH = 0x0000000e, + VS_PARTIAL_FLUSH = 0x0000000f, + PS_PARTIAL_FLUSH = 0x00000010, + FLUSH_HS_OUTPUT = 0x00000011, + FLUSH_DFSM = 0x00000012, + RESET_TO_LOWEST_VGT = 0x00000013, + CACHE_FLUSH_AND_INV_TS_EVENT = 0x00000014, + CACHE_FLUSH_AND_INV_EVENT = 0x00000016, + PERFCOUNTER_START = 0x00000017, + PERFCOUNTER_STOP = 0x00000018, + PIPELINESTAT_START = 0x00000019, + PIPELINESTAT_STOP = 0x0000001a, + PERFCOUNTER_SAMPLE = 0x0000001b, + SAMPLE_PIPELINESTAT = 0x0000001e, + SO_VGTSTREAMOUT_FLUSH = 0x0000001f, + SAMPLE_STREAMOUTSTATS = 0x00000020, + RESET_VTX_CNT = 0x00000021, + BLOCK_CONTEXT_DONE = 0x00000022, + CS_CONTEXT_DONE = 0x00000023, + VGT_FLUSH = 0x00000024, + TGID_ROLLOVER = 0x00000025, + SQ_NON_EVENT = 0x00000026, + SC_SEND_DB_VPZ = 0x00000027, + BOTTOM_OF_PIPE_TS = 0x00000028, + FLUSH_SX_TS = 0x00000029, + DB_CACHE_FLUSH_AND_INV = 0x0000002a, + FLUSH_AND_INV_DB_DATA_TS = 0x0000002b, + FLUSH_AND_INV_DB_META = 0x0000002c, + FLUSH_AND_INV_CB_DATA_TS = 0x0000002d, + FLUSH_AND_INV_CB_META = 0x0000002e, + CS_DONE = 0x0000002f, + PS_DONE = 0x00000030, + FLUSH_AND_INV_CB_PIXEL_DATA = 0x00000031, + SX_CB_RAT_ACK_REQUEST = 0x00000032, + THREAD_TRACE_START = 0x00000033, + THREAD_TRACE_STOP = 0x00000034, + THREAD_TRACE_MARKER = 0x00000035, + THREAD_TRACE_FINISH = 0x00000037, + PIXEL_PIPE_STAT_CONTROL = 0x00000038, + PIXEL_PIPE_STAT_DUMP = 0x00000039, + PIXEL_PIPE_STAT_RESET = 0x0000003a, + CONTEXT_SUSPEND = 0x0000003b, + OFFCHIP_HS_DEALLOC = 0x0000003c, + ENABLE_NGG_PIPELINE = 0x0000003d, + SET_FE_ID__GFX09 = 0x00000009, + Available_0x1c__GFX09 = 0x0000001c, + Available_0x1d__GFX09 = 0x0000001d, + THREAD_TRACE_FLUSH__GFX09 = 0x00000036, + Reserved_0x3f__GFX09 = 0x0000003f, + ZPASS_DONE__GFX09_10 = 0x00000015, + ENABLE_LEGACY_PIPELINE__GFX09_10 = 0x0000003e, + Reserved_0x09__GFX10PLUS = 0x00000009, + FLUSH_ES_OUTPUT__GFX10PLUS = 0x0000001c, + BIN_CONF_OVERRIDE_CHECK__GFX10PLUS = 0x0000001d, + THREAD_TRACE_DRAW__GFX10PLUS = 0x00000036, + DRAW_DONE__GFX10PLUS = 0x0000003f, + WAIT_SYNC__GFX11 = 0x00000015, + ENABLE_PIPELINE_NOT_USED__GFX11 = 0x0000003e, +}; + +struct PM4_MEC_EVENT_WRITE { + union { + PM4_MEC_TYPE_3_HEADER header; + uint32_t ordinal1; + }; + union { + struct { + uint32_t event_type:6; + uint32_t reserved1:2; + uint32_t event_index:4; + uint32_t reserved2:19; + uint32_t offload_enable:1; + } bitfields2; + uint32_t ordinal2; + }; +}; + +struct PM4_MEC_ATOMIC_MEM { + union { + PM4_MEC_TYPE_3_HEADER header; + uint32_t ordinal1; + }; + union { + struct { + uint32_t atomic:7; + uint32_t reserved1:1; + uint32_t command:4; + uint32_t reserved2:13; + uint32_t cache_policy:2; + uint32_t reserved3:5; + } bitfields2; + uint32_t ordinal2; + }; + uint32_t addr_lo; + uint32_t addr_hi; + uint32_t src_data_lo; + uint32_t src_data_hi; + uint32_t cmp_data_lo; + uint32_t cmp_data_hi; + union { + struct { + uint32_t loop_interval:13; + uint32_t reserved4:19; + } bitfields9; + uint32_t ordinal9; + }; +}; + +struct PM4_MEC_WRITE_DATA { + union { + PM4_MEC_TYPE_3_HEADER header; + uint32_t ordinal1; + }; + union { + struct { + uint32_t reserved1:8; + uint32_t dst_sel:4; + uint32_t reserved2:4; + uint32_t addr_incr:1; + uint32_t reserved3:2; + uint32_t resume_vf:1; + uint32_t wr_confirm:1; + uint32_t reserved4:4; + uint32_t cache_policy:2; + uint32_t reserved5:5; + } bitfields2; + uint32_t ordinal2; + }; + union { + struct { + uint32_t dst_mmreg_addr:18; + uint32_t reserved6:14; + } bitfields3a; + struct { + uint32_t dst_gds_addr:16; + uint32_t reserved7:16; + } bitfields3b; + struct { + uint32_t reserved8:2; + uint32_t dst_mem_addr_lo:30; + } bitfields3c; + uint32_t ordinal3; + }; + uint32_t dst_mem_addr_hi; +}; + +#define PERSISTENT_SPACE_START 0x00002c00 + +template +void GenerateSetShRegHeader(T* pm4, uint32_t reg_addr) { + pm4->cmd_set_data.header.u32All = PM4_TYPE3_HDR(IT_SET_SH_REG, + sizeof(T) / sizeof(uint32_t)); + pm4->cmd_set_data.bitfields2.reg_offset = reg_addr - PERSISTENT_SPACE_START; +} + +template +void GenerateCmdHeader(T* pm4, int op_code) { + pm4->header.u32All = PM4_TYPE3_HDR(op_code, sizeof(T) / sizeof(uint32_t)); +} + +/// @brief Defines the Gpu command to dispatch a kernel. It embeds +/// various Gpu hardware specific data structures for initialization +/// and configuration before a dispatch begins to run +struct DispatchTemplate { + + /// @brief Structure used to initialize the group dimensions + /// of a kernel dispatch and if performance counters are enabled + struct DispatchDimensionRegs { + PM4_MEC_SET_SH_REG cmd_set_data; + uint32_t compute_num_thread_x; + uint32_t compute_num_thread_y; + uint32_t compute_num_thread_z; + } dimension_regs; + + struct DispatchProgramRegs { + PM4_MEC_SET_SH_REG cmd_set_data; + uint32_t compute_pgm_lo; + uint32_t compute_pgm_hi; + } program_regs; + + struct DispatchProgramResourceRegs { + PM4_MEC_SET_SH_REG cmd_set_data; + uint32_t compute_pgm_rsrc1; + uint32_t compute_pgm_rsrc2; + } program_resource_regs; + + /// @brief Structure used to initialize parameters related to + /// thread management i.e. number of waves to issue and number + /// of Compute Units to use + struct DispatchResourceRegs { + PM4_MEC_SET_SH_REG cmd_set_data; + uint32_t compute_resource_limits; + uint32_t compute_static_thread_mgmt_se0; + uint32_t compute_static_thread_mgmt_se1; + uint32_t compute_tmpring_size; + uint32_t compute_static_thread_mgmt_se2; + uint32_t compute_static_thread_mgmt_se3; + } resource_regs; + + /// @brief Structure used to pass handles of the Aql dispatch + /// packet, Aql queue, Kernel argument address block, Scratch + /// buffer + struct DispatchComputeUserDataRegs { + PM4_MEC_SET_SH_REG cmd_set_data; + uint32_t compute_user_data[16]; + } compute_user_data_regs; + + /// @brief Structure used to configure Cache flush policy + /// and dimensions of total work size + PM4_MEC_DISPATCH_DIRECT dispatch_direct; +}; + +struct DispatchProgramResourceRegs { + PM4_MEC_SET_SH_REG cmd_set_data; + uint32_t compute_pgm_rsrc3; +}; + + +/// @brief Structure used to issue a programing scratch command for gfx11+ +struct SetScratchTemplate { + PM4_MEC_SET_SH_REG cmd_set_data; + uint32_t scratch_lo; + uint32_t scratch_hi; +}; + +/// @brief Structure used to issue a Gpu Barrier command +struct BarrierTemplate { + PM4_MEC_EVENT_WRITE event_write; +}; + +//--------------------MEC_ATOMIC_MEM-------------------- +enum MEC_ATOMIC_MEM_command_enum { + command__mec_atomic_mem__single_pass_atomic = 0, + command__mec_atomic_mem__loop_until_compare_satisfied = 1, + command__mec_atomic_mem__wait_for_write_confirmation = 2, + command__mec_atomic_mem__send_and_continue = 3, +}; + +enum MEC_ATOMIC_MEM_cache_policy_enum { + cache_policy__mec_atomic_mem__lru = 0, + cache_policy__mec_atomic_mem__stream = 1, + cache_policy__mec_atomic_mem__noa = 2, + cache_policy__mec_atomic_mem__bypass = 3, +}; + +enum TC_OP { + TC_OP_READ = 0x00000000, + TC_OP_ATOMIC_FCMPSWAP_RTN_32 = 0x00000001, + TC_OP_ATOMIC_FMIN_RTN_32 = 0x00000002, + TC_OP_ATOMIC_FMAX_RTN_32 = 0x00000003, + TC_OP_RESERVED_FOP_RTN_32_0 = 0x00000004, + TC_OP_RESERVED_FOP_RTN_32_2 = 0x00000006, + TC_OP_ATOMIC_SWAP_RTN_32 = 0x00000007, + TC_OP_ATOMIC_CMPSWAP_RTN_32 = 0x00000008, + TC_OP_ATOMIC_FCMPSWAP_FLUSH_DENORM_RTN_32 = 0x00000009, + TC_OP_ATOMIC_FMIN_FLUSH_DENORM_RTN_32 = 0x0000000a, + TC_OP_ATOMIC_FMAX_FLUSH_DENORM_RTN_32 = 0x0000000b, + TC_OP_PROBE_FILTER = 0x0000000c, + TC_OP_RESERVED_FOP_FLUSH_DENORM_RTN_32_2 = 0x0000000e, + TC_OP_ATOMIC_ADD_RTN_32 = 0x0000000f, + TC_OP_ATOMIC_SUB_RTN_32 = 0x00000010, + TC_OP_ATOMIC_SMIN_RTN_32 = 0x00000011, + TC_OP_ATOMIC_UMIN_RTN_32 = 0x00000012, + TC_OP_ATOMIC_SMAX_RTN_32 = 0x00000013, + TC_OP_ATOMIC_UMAX_RTN_32 = 0x00000014, + TC_OP_ATOMIC_AND_RTN_32 = 0x00000015, + TC_OP_ATOMIC_OR_RTN_32 = 0x00000016, + TC_OP_ATOMIC_XOR_RTN_32 = 0x00000017, + TC_OP_ATOMIC_INC_RTN_32 = 0x00000018, + TC_OP_ATOMIC_DEC_RTN_32 = 0x00000019, + TC_OP_WBINVL1_VOL = 0x0000001a, + TC_OP_WBINVL1_SD = 0x0000001b, + TC_OP_RESERVED_NON_FLOAT_RTN_32_0 = 0x0000001c, + TC_OP_RESERVED_NON_FLOAT_RTN_32_1 = 0x0000001d, + TC_OP_RESERVED_NON_FLOAT_RTN_32_2 = 0x0000001e, + TC_OP_RESERVED_NON_FLOAT_RTN_32_3 = 0x0000001f, + TC_OP_WRITE = 0x00000020, + TC_OP_ATOMIC_FCMPSWAP_RTN_64 = 0x00000021, + TC_OP_ATOMIC_FMIN_RTN_64 = 0x00000022, + TC_OP_ATOMIC_FMAX_RTN_64 = 0x00000023, + TC_OP_RESERVED_FOP_RTN_64_0 = 0x00000024, + TC_OP_RESERVED_FOP_RTN_64_1 = 0x00000025, + TC_OP_RESERVED_FOP_RTN_64_2 = 0x00000026, + TC_OP_ATOMIC_SWAP_RTN_64 = 0x00000027, + TC_OP_ATOMIC_CMPSWAP_RTN_64 = 0x00000028, + TC_OP_ATOMIC_FCMPSWAP_FLUSH_DENORM_RTN_64 = 0x00000029, + TC_OP_ATOMIC_FMIN_FLUSH_DENORM_RTN_64 = 0x0000002a, + TC_OP_ATOMIC_FMAX_FLUSH_DENORM_RTN_64 = 0x0000002b, + TC_OP_WBINVL2_SD = 0x0000002c, + TC_OP_RESERVED_FOP_FLUSH_DENORM_RTN_64_0 = 0x0000002d, + TC_OP_RESERVED_FOP_FLUSH_DENORM_RTN_64_1 = 0x0000002e, + TC_OP_ATOMIC_ADD_RTN_64 = 0x0000002f, + TC_OP_ATOMIC_SUB_RTN_64 = 0x00000030, + TC_OP_ATOMIC_SMIN_RTN_64 = 0x00000031, + TC_OP_ATOMIC_UMIN_RTN_64 = 0x00000032, + TC_OP_ATOMIC_SMAX_RTN_64 = 0x00000033, + TC_OP_ATOMIC_UMAX_RTN_64 = 0x00000034, + TC_OP_ATOMIC_AND_RTN_64 = 0x00000035, + TC_OP_ATOMIC_OR_RTN_64 = 0x00000036, + TC_OP_ATOMIC_XOR_RTN_64 = 0x00000037, + TC_OP_ATOMIC_INC_RTN_64 = 0x00000038, + TC_OP_ATOMIC_DEC_RTN_64 = 0x00000039, + TC_OP_WBL2_NC = 0x0000003a, + TC_OP_WBL2_WC = 0x0000003b, + TC_OP_RESERVED_NON_FLOAT_RTN_64_1 = 0x0000003c, + TC_OP_RESERVED_NON_FLOAT_RTN_64_2 = 0x0000003d, + TC_OP_RESERVED_NON_FLOAT_RTN_64_3 = 0x0000003e, + TC_OP_RESERVED_NON_FLOAT_RTN_64_4 = 0x0000003f, + TC_OP_WBINVL1 = 0x00000040, + TC_OP_ATOMIC_FCMPSWAP_32 = 0x00000041, + TC_OP_ATOMIC_FMIN_32 = 0x00000042, + TC_OP_ATOMIC_FMAX_32 = 0x00000043, + TC_OP_RESERVED_FOP_32_0 = 0x00000044, + TC_OP_RESERVED_FOP_32_2 = 0x00000046, + TC_OP_ATOMIC_SWAP_32 = 0x00000047, + TC_OP_ATOMIC_CMPSWAP_32 = 0x00000048, + TC_OP_ATOMIC_FCMPSWAP_FLUSH_DENORM_32 = 0x00000049, + TC_OP_ATOMIC_FMIN_FLUSH_DENORM_32 = 0x0000004a, + TC_OP_ATOMIC_FMAX_FLUSH_DENORM_32 = 0x0000004b, + TC_OP_INV_METADATA = 0x0000004c, + TC_OP_RESERVED_FOP_FLUSH_DENORM_32_2 = 0x0000004e, + TC_OP_ATOMIC_ADD_32 = 0x0000004f, + TC_OP_ATOMIC_SUB_32 = 0x00000050, + TC_OP_ATOMIC_SMIN_32 = 0x00000051, + TC_OP_ATOMIC_UMIN_32 = 0x00000052, + TC_OP_ATOMIC_SMAX_32 = 0x00000053, + TC_OP_ATOMIC_UMAX_32 = 0x00000054, + TC_OP_ATOMIC_AND_32 = 0x00000055, + TC_OP_ATOMIC_OR_32 = 0x00000056, + TC_OP_ATOMIC_XOR_32 = 0x00000057, + TC_OP_ATOMIC_INC_32 = 0x00000058, + TC_OP_ATOMIC_DEC_32 = 0x00000059, + TC_OP_INVL2_NC = 0x0000005a, + TC_OP_NOP_RTN0 = 0x0000005b, + TC_OP_RESERVED_NON_FLOAT_32_1 = 0x0000005c, + TC_OP_RESERVED_NON_FLOAT_32_2 = 0x0000005d, + TC_OP_RESERVED_NON_FLOAT_32_3 = 0x0000005e, + TC_OP_RESERVED_NON_FLOAT_32_4 = 0x0000005f, + TC_OP_WBINVL2 = 0x00000060, + TC_OP_ATOMIC_FCMPSWAP_64 = 0x00000061, + TC_OP_ATOMIC_FMIN_64 = 0x00000062, + TC_OP_ATOMIC_FMAX_64 = 0x00000063, + TC_OP_RESERVED_FOP_64_0 = 0x00000064, + TC_OP_RESERVED_FOP_64_1 = 0x00000065, + TC_OP_RESERVED_FOP_64_2 = 0x00000066, + TC_OP_ATOMIC_SWAP_64 = 0x00000067, + TC_OP_ATOMIC_CMPSWAP_64 = 0x00000068, + TC_OP_ATOMIC_FCMPSWAP_FLUSH_DENORM_64 = 0x00000069, + TC_OP_ATOMIC_FMIN_FLUSH_DENORM_64 = 0x0000006a, + TC_OP_ATOMIC_FMAX_FLUSH_DENORM_64 = 0x0000006b, + TC_OP_RESERVED_FOP_FLUSH_DENORM_64_0 = 0x0000006c, + TC_OP_RESERVED_FOP_FLUSH_DENORM_64_1 = 0x0000006d, + TC_OP_RESERVED_FOP_FLUSH_DENORM_64_2 = 0x0000006e, + TC_OP_ATOMIC_ADD_64 = 0x0000006f, + TC_OP_ATOMIC_SUB_64 = 0x00000070, + TC_OP_ATOMIC_SMIN_64 = 0x00000071, + TC_OP_ATOMIC_UMIN_64 = 0x00000072, + TC_OP_ATOMIC_SMAX_64 = 0x00000073, + TC_OP_ATOMIC_UMAX_64 = 0x00000074, + TC_OP_ATOMIC_AND_64 = 0x00000075, + TC_OP_ATOMIC_OR_64 = 0x00000076, + TC_OP_ATOMIC_XOR_64 = 0x00000077, + TC_OP_ATOMIC_INC_64 = 0x00000078, + TC_OP_ATOMIC_DEC_64 = 0x00000079, + TC_OP_WBINVL2_NC = 0x0000007a, + TC_OP_NOP_ACK = 0x0000007b, + TC_OP_RESERVED_NON_FLOAT_64_1 = 0x0000007c, + TC_OP_RESERVED_NON_FLOAT_64_2 = 0x0000007d, + TC_OP_RESERVED_NON_FLOAT_64_3 = 0x0000007e, + TC_OP_RESERVED_NON_FLOAT_64_4 = 0x0000007f, + TC_OP_RESERVED_FOP_RTN_32_1__GFX09_10 = 0x00000005, + TC_OP_RESERVED_FOP_FLUSH_DENORM_RTN_32_1__GFX09_10 = 0x0000000d, + TC_OP_RESERVED_FOP_32_1__GFX09_10 = 0x00000045, + TC_OP_RESERVED_FOP_FLUSH_DENORM_32_1__GFX09_10 = 0x0000004d, + TC_OP_RESERVED_FADD_RTN_32__GFX11 = 0x00000005, + TC_OP_ATOMIC_FADD_FLUSH_DENORM_RTN_32__GFX11 = 0x0000000d, + TC_OP_RESERVED_FADD_32__GFX11 = 0x00000045, + TC_OP_ATOMIC_FADD_FLUSH_DENORM_32__GFX11 = 0x0000004d, +}; + +// Desc: Strucuture used to perform various atomic +// operations - add, subtract, increment, etc +struct AtomicTemplate { + PM4_MEC_ATOMIC_MEM atomic; +}; + +/// @brief PM4 command to write a 64-bit value into a memory +/// location accessible to Gpu +struct WriteDataTemplate { + PM4_MEC_WRITE_DATA write_data; + uint64_t write_data_value; +}; + +// ---------------------------------- MEC_COPY_DATA_src_sel_enum ---------------------------------- +enum MEC_COPY_DATA_src_sel_enum { + src_sel__mec_copy_data__mem_mapped_register = 0, + src_sel__mec_copy_data__tc_l2_obsolete = 1, + src_sel__mec_copy_data__tc_l2 = 2, + src_sel__mec_copy_data__gds = 3, + src_sel__mec_copy_data__perfcounters = 4, + src_sel__mec_copy_data__immediate_data = 5, + src_sel__mec_copy_data__atomic_return_data = 6, + src_sel__mec_copy_data__gds_atomic_return_data0 = 7, + src_sel__mec_copy_data__gds_atomic_return_data1 = 8, + src_sel__mec_copy_data__gpu_clock_count = 9, + src_sel__mec_copy_data__system_clock_count = 10, + src_sel__mec_copy_data__ext32perfcntr = 11, +}; + +// ---------------------------------- MEC_COPY_DATA_dst_sel_enum ---------------------------------- +enum MEC_COPY_DATA_dst_sel_enum { + dst_sel__mec_copy_data__mem_mapped_register = 0, + dst_sel__mec_copy_data__tc_l2 = 2, + dst_sel__mec_copy_data__gds = 3, + dst_sel__mec_copy_data__perfcounters = 4, + dst_sel__mec_copy_data__tc_l2_obsolete = 5, + dst_sel__mec_copy_data__mem_mapped_reg_dc = 6, + dst_sel__mec_copy_data__ext32perfcntr = 11, +}; + +// ------------------------------ MEC_COPY_DATA_src_cache_policy_enum ------------------------------ +enum MEC_COPY_DATA_src_cache_policy_enum { + src_cache_policy__mec_copy_data__lru = 0, + src_cache_policy__mec_copy_data__stream = 1, + src_cache_policy__mec_copy_data__noa = 2, + src_cache_policy__mec_copy_data__bypass = 3, +}; + +// --------------------------------- MEC_COPY_DATA_count_sel_enum --------------------------------- +enum MEC_COPY_DATA_count_sel_enum { + count_sel__mec_copy_data__32_bits_of_data = 0, + count_sel__mec_copy_data__64_bits_of_data = 1, +}; + +// --------------------------------- MEC_COPY_DATA_wr_confirm_enum --------------------------------- +enum MEC_COPY_DATA_wr_confirm_enum { + wr_confirm__mec_copy_data__do_not_wait_for_confirmation = 0, + wr_confirm__mec_copy_data__wait_for_confirmation = 1, +}; + +// ------------------------------ MEC_COPY_DATA_dst_cache_policy_enum ------------------------------ +enum MEC_COPY_DATA_dst_cache_policy_enum { + dst_cache_policy__mec_copy_data__lru = 0, + dst_cache_policy__mec_copy_data__stream = 1, + dst_cache_policy__mec_copy_data__noa = 2, + dst_cache_policy__mec_copy_data__bypass = 3, +}; + +// ------------------------------- MEC_COPY_DATA_pq_exe_status_enum ------------------------------- +enum MEC_COPY_DATA_pq_exe_status_enum { + pq_exe_status__mec_copy_data__default = 0, + pq_exe_status__mec_copy_data__phase_update = 1, +}; + +typedef struct PM4_MEC_COPY_DATA { + union { + PM4_MEC_TYPE_3_HEADER header; /// header + uint32_t ordinal1; + }; + union { + struct { + uint32_t src_sel : 4; + uint32_t reserved1 : 4; + uint32_t dst_sel : 4; + uint32_t reserved2 : 1; + uint32_t src_cache_policy : 2; + uint32_t reserved3 : 1; + uint32_t count_sel : 1; + uint32_t reserved4 : 3; + uint32_t wr_confirm : 1; + uint32_t reserved5 : 4; + uint32_t dst_cache_policy : 2; + uint32_t reserved6 : 2; + uint32_t pq_exe_status : 1; + uint32_t reserved7 : 2; + } bitfields2; + uint32_t ordinal2; + }; + union { + struct { + uint32_t src_reg_offset : 18; + uint32_t reserved8 : 14; + } bitfields3a; + struct { + uint32_t reserved9 : 2; + uint32_t src_32b_addr_lo : 30; + } bitfields3b; + struct { + uint32_t reserved10 : 3; + uint32_t src_64b_addr_lo : 29; + } bitfields3c; + struct { + uint32_t src_gds_addr_lo : 16; + uint32_t reserved11 : 16; + } bitfields3d; + uint32_t imm_data; + uint32_t ordinal3; + }; + union { + uint32_t src_memtc_addr_hi; + uint32_t src_imm_data; + uint32_t ordinal4; + }; + union { + struct { + uint32_t dst_reg_offset : 18; + uint32_t reserved12 : 14; + } bitfields5a; + struct { + uint32_t reserved13 : 2; + uint32_t dst_32b_addr_lo : 30; + } bitfields5b; + struct { + uint32_t reserved14 : 3; + uint32_t dst_64b_addr_lo : 29; + } bitfields5c; + struct { + uint32_t dst_gds_addr_lo : 16; + uint32_t reserved15 : 16; + } bitfields5d; + uint32_t ordinal5; + }; + uint32_t dst_addr_hi; +} PM4MEC_COPY_DATA; +namespace gfx9 { + +struct PM4_MEC_ACQUIRE_MEM { + union { + PM4_MEC_TYPE_3_HEADER header; + uint32_t ordinal1; + }; + union { + struct { + uint32_t coher_cntl:31; + uint32_t reserved1:1; + } bitfields2; + uint32_t ordinal2; + }; + uint32_t coher_size; + union { + struct { + uint32_t coher_size_hi:8; + uint32_t reserved2:24; + } bitfields4; + uint32_t ordinal4; + }; + uint32_t coher_base_lo; + union { + struct { + uint32_t coher_base_hi:24; + uint32_t reserved3:8; + } bitfields6; + uint32_t ordinal6; + }; + union { + struct { + uint32_t poll_interval:16; + uint32_t reserved4:16; + } bitfields7; + uint32_t ordinal7; + }; +}; + +struct PM4_MEC_RELEASE_MEM { + union { + PM4_MEC_TYPE_3_HEADER header; + uint32_t ordinal1; + }; + union { + struct { + uint32_t event_type:6; + uint32_t reserved1:2; + uint32_t event_index:4; + uint32_t tcl1_vol_action_ena:1; + uint32_t tc_vol_action_ena:1; + uint32_t reserved2:1; + uint32_t tc_wb_action_ena:1; + uint32_t tcl1_action_ena:1; + uint32_t tc_action_ena:1; + uint32_t reserved3:1; + uint32_t tc_nc_action_ena:1; + uint32_t tc_wc_action_ena:1; + uint32_t tc_md_action_ena:1; + uint32_t reserved4:3; + uint32_t cache_policy:2; + uint32_t reserved5:2; + uint32_t pq_exe_status:1; + uint32_t reserved6:2; + } bitfields2; + uint32_t ordinal2; + }; + union { + struct { + uint32_t reserved7:16; + uint32_t dst_sel:2; + uint32_t reserved8:6; + uint32_t int_sel:3; + uint32_t reserved9:2; + uint32_t data_sel:3; + } bitfields3; + uint32_t ordinal3; + }; + union { + struct { + uint32_t reserved10:2; + uint32_t address_lo_32b:30; + } bitfields4a; + struct { + uint32_t reserved11:3; + uint32_t address_lo_64b:29; + } bitfields4b; + uint32_t reserved12; + uint32_t ordinal4; + }; + union { + uint32_t address_hi; + uint32_t reserved13; + uint32_t ordinal5; + }; + union { + uint32_t data_lo; + uint32_t cmp_data_lo; + struct { + uint32_t dw_offset:16; + uint32_t num_dwords:16; + } bitfields6c; + uint32_t reserved14; + uint32_t ordinal6; + }; + union { + uint32_t data_hi; + uint32_t cmp_data_hi; + uint32_t reserved15; + uint32_t reserved16; + uint32_t ordinal7; + }; + uint32_t int_ctxid; +}; + +struct PM4_MEC_WAIT_REG_MEM64 { + union { + PM4_MEC_TYPE_3_HEADER header; + uint32_t ordinal1; + }; + union { + struct { + uint32_t function:3; + uint32_t reserved1:1; + uint32_t mem_space:2; + uint32_t operation:2; + uint32_t reserved2:24; + } bitfields2; + uint32_t ordinal2; + }; + union { + struct { + uint32_t reserved3:3; + uint32_t mem_poll_addr_lo:29; + } bitfields3a; + struct { + uint32_t reg_poll_addr:18; + uint32_t reserved4:14; + } bitfields3b; + struct { + uint32_t reg_write_addr1:18; + uint32_t reserved5:14; + } bitfields3c; + uint32_t ordinal3; + }; + union { + uint32_t mem_poll_addr_hi; + struct { + uint32_t reg_write_addr2:18; + uint32_t reserved6:14; + } bitfields4b; + uint32_t ordinal4; + }; + uint32_t reference; + uint32_t reference_hi; + uint32_t mask; + uint32_t mask_hi; + union { + struct { + uint32_t poll_interval:16; + uint32_t reserved7:16; + } bitfields9; + uint32_t ordinal9; + }; +}; + +/// @brief Structure used to configure the flushing of +/// various caches - instruction, constants, L1 and L2 +struct AcquireMemTemplate { + PM4_MEC_ACQUIRE_MEM acquire_mem; +}; + +struct EndofKernelNotifyTemplate { + PM4_MEC_RELEASE_MEM release_mem; +}; + +/// @brief PM4 command to wait for a certain event before proceeding +/// to process another command on the queue +struct WaitRegMem64Template { + PM4_MEC_WAIT_REG_MEM64 wait_reg_mem; +}; + +} // gfx9 namespace + +namespace gfx10 { + +struct PM4_MEC_ACQUIRE_MEM { + union { + PM4_MEC_TYPE_3_HEADER header; + uint32_t ordinal1; + }; + uint32_t reserved1; + uint32_t coher_size; + union { + struct { + uint32_t coher_size_hi:8; + uint32_t reserved2:24; + } bitfields4; + uint32_t ordinal4; + }; + uint32_t coher_base_lo; + union { + struct { + uint32_t coher_base_hi:24; + uint32_t reserved3:8; + } bitfields6; + uint32_t ordinal6; + }; + union { + struct { + uint32_t poll_interval:16; + uint32_t reserved4:16; + } bitfields7; + uint32_t ordinal7; + }; + union { + struct { + uint32_t gcr_cntl:19; + uint32_t reserved4:13; + } bitfields8; + uint32_t ordinal8; + }; +}; + +struct PM4_MEC_RELEASE_MEM { + union { + PM4_MEC_TYPE_3_HEADER header; + uint32_t ordinal1; + }; + union { + struct { + uint32_t event_type:6; + uint32_t reserved1:2; + uint32_t event_index:4; + uint32_t gcr_cntl:12; + uint32_t reserved2:1; + uint32_t cache_policy:2; + uint32_t reserved3:2; + uint32_t pq_exe_status:1; + uint32_t reserved4:2; + } bitfields2; + uint32_t ordinal2; + }; + union { + struct { + uint32_t reserved7:16; + uint32_t dst_sel:2; + uint32_t reserved8:2; + uint32_t mes_intr_pipe:2; + uint32_t mes_action_id:2; + uint32_t int_sel:3; + uint32_t reserved9:2; + uint32_t data_sel:3; + } bitfields3; + uint32_t ordinal3; + }; + union { + struct { + uint32_t reserved10:2; + uint32_t address_lo_32b:30; + } bitfields4a; + struct { + uint32_t reserved11:3; + uint32_t address_lo_64b:29; + } bitfields4b; + uint32_t reserved12; + uint32_t ordinal4; + }; + union { + uint32_t address_hi; + uint32_t reserved13; + uint32_t ordinal5; + }; + union { + uint32_t data_lo; + uint32_t cmp_data_lo; + struct { + uint32_t dw_offset:16; + uint32_t num_dwords:16; + } bitfields6c; + uint32_t reserved14; + uint32_t ordinal6; + }; + union { + uint32_t data_hi; + uint32_t cmp_data_hi; + uint32_t reserved15; + uint32_t reserved16; + uint32_t ordinal7; + }; + uint32_t int_ctxid; +}; + +struct PM4_MEC_WAIT_REG_MEM64 { + union { + PM4_MEC_TYPE_3_HEADER header; ///header + uint32_t ordinal1; + }; + union { + struct { + uint32_t function:3; + uint32_t reserved1:1; + uint32_t mem_space:2; + uint32_t operation:2; + uint32_t reserved2:14; + uint32_t mes_intr_pipe:2; + uint32_t mes_action:1; + uint32_t cache_policy:2; + uint32_t reserved3:5; + } bitfields2; + uint32_t ordinal2; + }; + union { + struct { + uint32_t reserved4:3; + uint32_t mem_poll_addr_lo:29; + } bitfields3a; + struct { + uint32_t reg_poll_addr:18; + uint32_t reserved5:14; + } bitfields3b; + struct { + uint32_t reg_write_addr1:18; + uint32_t reserved6:14; + } bitfields3c; + uint32_t ordinal3; + }; + union { + uint32_t mem_poll_addr_hi; + struct { + uint32_t reg_write_addr2:18; + uint32_t reserved7:14; + } bitfields4b; + uint32_t ordinal4; + }; + uint32_t reference; + uint32_t reference_hi; + uint32_t mask; + uint32_t mask_hi; + union { + struct { + uint32_t poll_interval:16; + uint32_t reserved8:15; + uint32_t optimize_ace_offload_mode:1; + } bitfields9; + uint32_t ordinal9; + }; +}; + +/// @brief Structure used to configure the flushing of +/// various caches - instruction, constants, L1 and L2 +struct AcquireMemTemplate { + PM4_MEC_ACQUIRE_MEM acquire_mem; +}; + +struct EndofKernelNotifyTemplate { + PM4_MEC_RELEASE_MEM release_mem; +}; + +struct WaitRegMem64Template { + PM4_MEC_WAIT_REG_MEM64 wait_reg_mem; +}; + +} // gfx10 namespace + +namespace gfx11 { + +struct PM4_MEC_RELEASE_MEM { + union { + PM4_MEC_TYPE_3_HEADER header; + uint32_t ordinal1; + }; + union { + struct { + uint32_t event_type:6; + uint32_t reserved1:2; + uint32_t event_index:4; + uint32_t gcr_cntl:13; + uint32_t cache_policy:2; + uint32_t reserved2:1; + uint32_t pq_exe_status:1; + uint32_t reserved3:1; + uint32_t glk_inv:1; + uint32_t reserved4:1; + } bitfields2; + uint32_t ordinal2; + }; + union { + struct { + uint32_t reserved5:16; + uint32_t dst_sel:2; + uint32_t reserved6:2; + uint32_t mes_intr_pipe:2; + uint32_t mes_action_id:2; + uint32_t int_sel:3; + uint32_t reserved7:2; + uint32_t data_sel:3; + } bitfields3; + uint32_t ordinal3; + }; + union { + struct { + uint32_t reserved8:2; + uint32_t address_lo_32b:30; + } bitfields4a; + struct { + uint32_t reserved9:3; + uint32_t address_lo_64b:29; + } bitfields4b; + uint32_t reserved10; + uint32_t ordinal4; + }; + union { + uint32_t address_hi; + uint32_t reserved11; + uint32_t ordinal5; + }; + union { + uint32_t data_lo; + uint32_t cmp_data_lo; + struct { + uint32_t dw_offset:16; + uint32_t num_dwords:16; + } bitfields6c; + uint32_t reserved12; + uint32_t ordinal6; + }; + union { + uint32_t data_hi; + uint32_t cmp_data_hi; + uint32_t reserved13; + uint32_t reserved14; + uint32_t ordinal7; + }; + uint32_t int_ctxid; +}; + +struct EndofKernelNotifyTemplate { + PM4_MEC_RELEASE_MEM release_mem; +}; + +} // gfx11 namespace + +#endif diff --git a/inc/registers.h b/inc/registers.h new file mode 100644 index 0000000000..067b808b90 --- /dev/null +++ b/inc/registers.h @@ -0,0 +1,363 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +// This file is used only for open source cmake builds, if we hardcode the +// register values in amd_aql_queue.cpp then this file won't be required. For +// now we are using this file where register details are spelled out in the +// structs/unions below. +#ifndef HSA_RUNTME_CORE_INC_REGISTERS_H_ +#define HSA_RUNTME_CORE_INC_REGISTERS_H_ + +typedef enum SQ_RSRC_BUF_TYPE { +SQ_RSRC_BUF = 0x00000000, +SQ_RSRC_BUF_RSVD_1 = 0x00000001, +SQ_RSRC_BUF_RSVD_2 = 0x00000002, +SQ_RSRC_BUF_RSVD_3 = 0x00000003, +} SQ_RSRC_BUF_TYPE; + +typedef enum BUF_DATA_FORMAT { +BUF_DATA_FORMAT_INVALID = 0x00000000, +BUF_DATA_FORMAT_8 = 0x00000001, +BUF_DATA_FORMAT_16 = 0x00000002, +BUF_DATA_FORMAT_8_8 = 0x00000003, +BUF_DATA_FORMAT_32 = 0x00000004, +BUF_DATA_FORMAT_16_16 = 0x00000005, +BUF_DATA_FORMAT_10_11_11 = 0x00000006, +BUF_DATA_FORMAT_11_11_10 = 0x00000007, +BUF_DATA_FORMAT_10_10_10_2 = 0x00000008, +BUF_DATA_FORMAT_2_10_10_10 = 0x00000009, +BUF_DATA_FORMAT_8_8_8_8 = 0x0000000a, +BUF_DATA_FORMAT_32_32 = 0x0000000b, +BUF_DATA_FORMAT_16_16_16_16 = 0x0000000c, +BUF_DATA_FORMAT_32_32_32 = 0x0000000d, +BUF_DATA_FORMAT_32_32_32_32 = 0x0000000e, +BUF_DATA_FORMAT_RESERVED_15 = 0x0000000f, +} BUF_DATA_FORMAT; + +typedef enum BUF_NUM_FORMAT { +BUF_NUM_FORMAT_UNORM = 0x00000000, +BUF_NUM_FORMAT_SNORM = 0x00000001, +BUF_NUM_FORMAT_USCALED = 0x00000002, +BUF_NUM_FORMAT_SSCALED = 0x00000003, +BUF_NUM_FORMAT_UINT = 0x00000004, +BUF_NUM_FORMAT_SINT = 0x00000005, +BUF_NUM_FORMAT_SNORM_OGL__SI__CI = 0x00000006, +BUF_NUM_FORMAT_RESERVED_6__VI = 0x00000006, +BUF_NUM_FORMAT_FLOAT = 0x00000007, +} BUF_NUM_FORMAT; + +typedef enum BUF_FORMAT { +BUF_FORMAT_32_UINT = 0x00000014, +} BUF_FORMAT; + +typedef enum SQ_SEL_XYZW01 { +SQ_SEL_0 = 0x00000000, +SQ_SEL_1 = 0x00000001, +SQ_SEL_RESERVED_0 = 0x00000002, +SQ_SEL_RESERVED_1 = 0x00000003, +SQ_SEL_X = 0x00000004, +SQ_SEL_Y = 0x00000005, +SQ_SEL_Z = 0x00000006, +SQ_SEL_W = 0x00000007, +} SQ_SEL_XYZW01; + + union COMPUTE_TMPRING_SIZE { + struct { +#if defined(LITTLEENDIAN_CPU) + unsigned int WAVES : 12; + unsigned int WAVESIZE : 13; + unsigned int : 7; +#elif defined(BIGENDIAN_CPU) + unsigned int : 7; + unsigned int WAVESIZE : 13; + unsigned int WAVES : 12; +#endif + } bitfields, bits; + unsigned int u32All; + signed int i32All; + float f32All; + }; + + union COMPUTE_TMPRING_SIZE_GFX11 { + struct { +#if defined(LITTLEENDIAN_CPU) + unsigned int WAVES : 12; + unsigned int WAVESIZE : 15; + unsigned int : 5; +#elif defined(BIGENDIAN_CPU) + unsigned int : 5; + unsigned int WAVESIZE : 15; + unsigned int WAVES : 12; +#endif + } bitfields, bits; + unsigned int u32All; + signed int i32All; + float f32All; + }; + + union COMPUTE_TMPRING_SIZE_GFX12 { + struct { +#if defined(LITTLEENDIAN_CPU) + unsigned int WAVES : 12; + unsigned int WAVESIZE : 18; + unsigned int : 2; +#elif defined(BIGENDIAN_CPU) + unsigned int : 2; + unsigned int WAVESIZE : 18; + unsigned int WAVES : 12; +#endif + } bitfields, bits; + unsigned int u32All; + signed int i32All; + float f32All; + }; + + union SQ_BUF_RSRC_WORD0 { + struct { +#if defined(LITTLEENDIAN_CPU) + unsigned int BASE_ADDRESS : 32; +#elif defined(BIGENDIAN_CPU) + unsigned int BASE_ADDRESS : 32; +#endif + } bitfields, bits; + unsigned int u32All; + signed int i32All; + float f32All; + }; + + + union SQ_BUF_RSRC_WORD1 { + struct { +#if defined(LITTLEENDIAN_CPU) + unsigned int BASE_ADDRESS_HI : 16; + unsigned int STRIDE : 14; + unsigned int CACHE_SWIZZLE : 1; + unsigned int SWIZZLE_ENABLE : 1; +#elif defined(BIGENDIAN_CPU) + unsigned int SWIZZLE_ENABLE : 1; + unsigned int CACHE_SWIZZLE : 1; + unsigned int STRIDE : 14; + unsigned int BASE_ADDRESS_HI : 16; +#endif + } bitfields, bits; + unsigned int u32All; + signed int i32All; + float f32All; + }; + + union SQ_BUF_RSRC_WORD1_GFX11 { + struct { +#if defined(LITTLEENDIAN_CPU) + unsigned int BASE_ADDRESS_HI : 16; + unsigned int STRIDE : 14; + unsigned int SWIZZLE_ENABLE : 2; +#elif defined(BIGENDIAN_CPU) + unsigned int SWIZZLE_ENABLE : 2; + unsigned int STRIDE : 14; + unsigned int BASE_ADDRESS_HI : 16; +#endif + } bitfields, bits; + unsigned int u32All; + signed int i32All; + float f32All; + }; + + + union SQ_BUF_RSRC_WORD2 { + struct { +#if defined(LITTLEENDIAN_CPU) + unsigned int NUM_RECORDS : 32; +#elif defined(BIGENDIAN_CPU) + unsigned int NUM_RECORDS : 32; +#endif + } bitfields, bits; + unsigned int u32All; + signed int i32All; + float f32All; + }; + + + union SQ_BUF_RSRC_WORD3 { + struct { +#if defined(LITTLEENDIAN_CPU) + unsigned int DST_SEL_X : 3; + unsigned int DST_SEL_Y : 3; + unsigned int DST_SEL_Z : 3; + unsigned int DST_SEL_W : 3; + unsigned int NUM_FORMAT : 3; + unsigned int DATA_FORMAT : 4; + unsigned int ELEMENT_SIZE : 2; + unsigned int INDEX_STRIDE : 2; + unsigned int ADD_TID_ENABLE : 1; + unsigned int ATC__CI__VI : 1; + unsigned int HASH_ENABLE : 1; + unsigned int HEAP : 1; + unsigned int MTYPE__CI__VI : 3; + unsigned int TYPE : 2; +#elif defined(BIGENDIAN_CPU) + unsigned int TYPE : 2; + unsigned int MTYPE__CI__VI : 3; + unsigned int HEAP : 1; + unsigned int HASH_ENABLE : 1; + unsigned int ATC__CI__VI : 1; + unsigned int ADD_TID_ENABLE : 1; + unsigned int INDEX_STRIDE : 2; + unsigned int ELEMENT_SIZE : 2; + unsigned int DATA_FORMAT : 4; + unsigned int NUM_FORMAT : 3; + unsigned int DST_SEL_W : 3; + unsigned int DST_SEL_Z : 3; + unsigned int DST_SEL_Y : 3; + unsigned int DST_SEL_X : 3; +#endif + } bitfields, bits; + unsigned int u32All; + signed int i32All; + float f32All; + }; + + union SQ_BUF_RSRC_WORD3_GFX10 { + struct { +#if defined(LITTLEENDIAN_CPU) + unsigned int DST_SEL_X : 3; + unsigned int DST_SEL_Y : 3; + unsigned int DST_SEL_Z : 3; + unsigned int DST_SEL_W : 3; + unsigned int FORMAT : 7; + unsigned int RESERVED1 : 2; + unsigned int INDEX_STRIDE : 2; + unsigned int ADD_TID_ENABLE : 1; + unsigned int RESOURCE_LEVEL : 1; + unsigned int RESERVED2 : 3; + unsigned int OOB_SELECT : 2; + unsigned int TYPE : 2; +#elif defined(BIGENDIAN_CPU) + unsigned int TYPE : 2; + unsigned int OOB_SELECT : 2; + unsigned int RESERVED2 : 3; + unsigned int RESOURCE_LEVEL : 1; + unsigned int ADD_TID_ENABLE : 1; + unsigned int INDEX_STRIDE : 2; + unsigned int RESERVED1 : 2; + unsigned int FORMAT : 7; + unsigned int DST_SEL_W : 3; + unsigned int DST_SEL_Z : 3; + unsigned int DST_SEL_Y : 3; + unsigned int DST_SEL_X : 3; +#endif + } bitfields, bits; + unsigned int u32All; + signed int i32All; + float f32All; + }; + + // From V# Table + union SQ_BUF_RSRC_WORD3_GFX11 { + struct { +#if defined(LITTLEENDIAN_CPU) + unsigned int DST_SEL_X : 3; + unsigned int DST_SEL_Y : 3; + unsigned int DST_SEL_Z : 3; + unsigned int DST_SEL_W : 3; + unsigned int FORMAT : 6; + unsigned int RESERVED1 : 3; + unsigned int INDEX_STRIDE : 2; + unsigned int ADD_TID_ENABLE : 1; + unsigned int RESERVED2 : 4; + unsigned int OOB_SELECT : 2; + unsigned int TYPE : 2; +#elif defined(BIGENDIAN_CPU) + unsigned int TYPE : 2; + unsigned int OOB_SELECT : 2; + unsigned int RESERVED2 : 4; + unsigned int ADD_TID_ENABLE : 1; + unsigned int INDEX_STRIDE : 2; + unsigned int RESERVED1 : 3; + unsigned int FORMAT : 6; + unsigned int DST_SEL_W : 3; + unsigned int DST_SEL_Z : 3; + unsigned int DST_SEL_Y : 3; + unsigned int DST_SEL_X : 3; +#endif + } bitfields, bits; + unsigned int u32All; + signed int i32All; + float f32All; + }; + // From V# Table + union SQ_BUF_RSRC_WORD3_GFX12 { + struct { +#if defined(LITTLEENDIAN_CPU) + unsigned int DST_SEL_X : 3; + unsigned int DST_SEL_Y : 3; + unsigned int DST_SEL_Z : 3; + unsigned int DST_SEL_W : 3; + unsigned int FORMAT : 6; + unsigned int RESERVED1 : 3; + unsigned int INDEX_STRIDE : 2; + unsigned int ADD_TID_ENABLE : 1; + unsigned int WRITE_COMPRESS_ENABLE : 1; + unsigned int COMPRESSION_EN : 1; + unsigned int COMPRESSION_ACCESS_MODE : 2; + unsigned int OOB_SELECT : 2; + unsigned int TYPE : 2; +#elif defined(BIGENDIAN_CPU) + unsigned int TYPE : 2; + unsigned int OOB_SELECT : 2; + unsigned int COMPRESSION_ACCESS_MODE : 2; + unsigned int COMPRESSION_EN : 1; + unsigned int WRITE_COMPRESS_ENABLE : 1; + unsigned int ADD_TID_ENABLE : 1; + unsigned int INDEX_STRIDE : 2; + unsigned int RESERVED1 : 3; + unsigned int FORMAT : 6; + unsigned int DST_SEL_W : 3; + unsigned int DST_SEL_Z : 3; + unsigned int DST_SEL_Y : 3; + unsigned int DST_SEL_X : 3; +#endif + } bitfields, bits; + unsigned int u32All; + signed int i32All; + float f32All; + }; +#endif // header guard diff --git a/inc/rocr_proxy/rocr_proxy.h b/inc/rocr_proxy/rocr_proxy.h new file mode 100644 index 0000000000..60358cdb90 --- /dev/null +++ b/inc/rocr_proxy/rocr_proxy.h @@ -0,0 +1,131 @@ +#ifndef _ROCR_PROXY_H_ +#define _ROCR_PROXY_H_ + +#include + +namespace rocr_proxy { +enum AllocDomain { + kSystem, + kLocal, + kUserMemory, + kUserQueue, + kDomainCount, +}; + +enum MemFlag { + kFineGrain = (1ULL << 0), + kKernarg = (1ULL << 1), +}; + +enum EngineFlag { + KCOMPUTE0 = (1ULL << 0), + KDRMDMA = (1ULL << 1), + KDRMDMA1 = (1ULL << 2), +}; + +enum SchedLevel { + kLow = 0, + kNormal = 1, + kHigh = 2, +}; + +enum AsicFamilyType { + kVega10, + kNavi10, + kSiennaCichlid, + kPlumBONITO, + kNavi44, + kNavi48 +}; + +struct HwsInfo { + union { + struct { + uint32_t gfxHwsEnabled : 1; + uint32_t computeHwsEnabled : 1; + uint32_t dmaHwsEnabled : 1; + uint32_t dma1HwsEnabled : 1; + uint32_t reserved : 28; + } hwsMask; + uint32_t osHwsEnableFlags; + }; + uint64_t engineOrdinalMask; // Indicates which engines (by ordinal) support MES HWS +}; + +typedef struct { + int major; + int minor; + int stepping; + bool is_dgpu; + char product_name[MAX_PATH]; + const char *uuid; + AsicFamilyType family; + uint32_t device_id; + uint32_t wavefront_size; + uint32_t compute_unit_count; + uint32_t max_engine_clock_mhz; + uint32_t watch_points_num; + uint32_t pci_bus_addr; + uint32_t memory_bus_width; + uint32_t max_memory_clock_mhz; + uint64_t gpu_counter_frequency; + uint32_t wave_per_cu; + uint32_t simd_per_cu; + uint32_t max_scratch_slots_per_cu; + uint32_t num_shader_engine; + uint32_t shader_array_per_shader_engine; + uint32_t domain; + uint32_t num_gws; + uint32_t asic_revision; + uint64_t local_visible_heap_size; + uint64_t local_invisible_heap_size; + uint64_t private_aperture_base; + uint64_t private_aperture_size; + uint64_t shared_aperture_base; + uint64_t shared_aperture_size; + uint32_t user_queue_size; + uint32_t lds_size; + uint32_t big_page_alignment_size; + uint32_t hw_big_page_min_alignment_size; + uint32_t hw_big_page_alignment_size; + bool enable_big_page_alignment; + uint32_t mec_fw_version; + uint32_t sdma_fw_version; + uint32_t l1_cache_size; + uint32_t l2_cache_size; + uint32_t l3_cache_size; + uint32_t gl2_cacheline_size; + uint32_t num_cp_queues; + HwsInfo hwsInfo; + std::vector sdma_schedid; + uint32_t compute_schedid; + bool state_shadowing_by_cpfw; + bool platform_atomic_support; + void *adapter_info; + void *adapter_ex_info; +} DeviceInfo; + +int EngineOrdinal(int engine, DeviceInfo *device_info); +bool GetHwsEnabled(int engine, DeviceInfo *device_info); +bool ShouldDisableGpuTimeout(int engine, DeviceInfo *device_info); +bool ParseAdapterInfo(D3DKMT_HANDLE adapter, DeviceInfo *device_info); +bool QueryAdapterSupported(D3DKMT_HANDLE adapter); + +uint32_t QueueEngine2EngineFlag(uint32_t queue_engine); +void SetAllocationInfo(void *data, uint64_t size, AllocDomain domain, + uint64_t addr, uint32_t mem_flags, uint32_t engine_flag, const DeviceInfo &device_info); +bool CreatePrivateAllocInfo(int num_handles, void **ppdrv_priv, void **ppalloc_priv, + int *pdrv_priv_data_size, int *palloc_priv_data_size); +void DestroyPrivateAllocInfo(void *drv_priv, void *alloc_priv); + +int CreateSubmitPrivData(void **priv_data, D3DKMT_HANDLE queue, uint64_t command_addr, + uint64_t command_size, bool is_hw_queue); +int CreateHwQueuePrivData(void **priv_data, D3DKMT_HANDLE context, + bool FwManagedGfxState, SchedLevel level = kNormal); +int CreateContextPrivData(void **priv_data, bool FwManagedGfxState); +int CreatePowerOptPrivData(void **priv_data, bool restore); +int CreateCalibratedTimestampsPrivData(void **priv_data); +void QueryCalibratedTimestamps(void* priv, uint64_t* gpu, uint64_t* cpu); +void DestroyPrivData(void *priv_data); +} +#endif diff --git a/inc/rocr_proxy/wddm_types.h b/inc/rocr_proxy/wddm_types.h new file mode 100644 index 0000000000..f2e60d907d --- /dev/null +++ b/inc/rocr_proxy/wddm_types.h @@ -0,0 +1,155 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2014-2015, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +#ifndef _ROCR_WDDM_TYPES_H_ +#define _ROCR_WDDM_TYPES_H_ + +#include + +#include + +typedef uint32_t UINT, *UINT_PTR; +typedef int32_t INT32; +typedef int32_t LONG; +typedef uint32_t ULONG, *ULONG_PTR; +typedef int64_t LONGLONG; +typedef int64_t LONG64; +typedef uint64_t ULONGLONG; +typedef uint64_t ULONG64, *ULONG64_PTR; +typedef uint8_t BYTE; +typedef uint16_t WORD; +typedef uint32_t DWORD; +typedef int32_t BOOL; +typedef int32_t NTSTATUS; +typedef uint16_t USHORT; +typedef uint16_t UINT16; +typedef uint32_t UINT32; +typedef uint64_t UINT64; +typedef int32_t INT; +typedef uint64_t SIZE_T; +typedef void VOID; +typedef float FLOAT; +typedef char CHAR; +typedef unsigned char UCHAR; +typedef UCHAR BOOLEAN; +typedef int16_t WCHAR; +typedef void *HANDLE; +typedef void *PVOID; +typedef void *LPVOID; +typedef const int16_t *PCWSTR; + +#define ULONG ULONG +#define ULONG_PTR ULONG_PTR +#define USHORT USHORT + +#define DECLARE_HANDLE(name) struct name##__{int unused;}; typedef struct name##__ *name +#define C_ASSERT(e) typedef char __C_ASSERT__[(e)?1:-1] + +DECLARE_HANDLE(HWND); +DECLARE_HANDLE(HDC); +DECLARE_HANDLE(PALETTEENTRY); + +typedef struct tagPOINT { + LONG x; + LONG y; +} POINT; + +typedef struct tagRECT { + LONG left; + LONG top; + LONG right; + LONG bottom; +} RECT; + +typedef struct tagRECTL { + LONG left; + LONG top; + LONG right; + LONG bottom; +} RECTL; + +typedef union _LARGE_INTEGER { + struct { + DWORD LowPart; + DWORD HighPart; + } u; + LONGLONG QuadPart; +} LARGE_INTEGER; + +typedef LARGE_INTEGER *PLARGE_INTEGER; + +typedef struct _LUID { + ULONG LowPart; + LONG HighPart; +} LUID, *PLUID; + +typedef enum _DEVICE_POWER_STATE { + PowerDeviceUnspecified = 0, + PowerDeviceD0, + PowerDeviceD1, + PowerDeviceD2, + PowerDeviceD3, + PowerDeviceMaximum +} DEVICE_POWER_STATE, *PDEVICE_POWER_STATE; + +#define _Check_return_ +#define APIENTRY +#define CONST const +#define IN +#define OUT +#define FAR +#define MAX_PATH 260 +#define __stdcall + +#ifndef GUID_DEFINED +#define GUID_DEFINED +typedef struct _GUID { + uint32_t Data1; + uint16_t Data2; + uint16_t Data3; + uint8_t Data4[ 8 ]; +} GUID; +#endif + +#include + +#endif diff --git a/inc/wddm/cmd_util.h b/inc/wddm/cmd_util.h new file mode 100644 index 0000000000..423d9d6290 --- /dev/null +++ b/inc/wddm/cmd_util.h @@ -0,0 +1,77 @@ +/* Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. */ + +#ifndef _CMD_UTIL_H_ +#define _CMD_UTIL_H_ + +#include +#include "hsa-runtime/inc/hsa.h" +#include "hsa-runtime/inc/amd_hsa_queue.h" +#include "hsa-runtime/inc/amd_hsa_kernel_code.h" +#include "inc/pm4_cmds.h" +#include "util/utils.h" + +namespace rocr { +namespace core { + +struct DispatchInfo { + uint8_t major; + hsa_kernel_dispatch_packet_t *pPacket; + void *pEntry; + const amd_kernel_code_t *pKernelObject; + uint32_t ldsBlks; + amd_queue_t *pAmdQueue; + bool wave32; + uint32_t srd; + void *pScratchBase; + uint32_t scratchSizePerWave; + uint32_t scratchBaseOffset[2]; + uint32_t offsetCnt; +}; + +class CmdUtil { +public: + CmdUtil() {}; + ~CmdUtil() {}; + + size_t BuildCopyData( + uint64_t *pDstAddr, + void *pBuffer, + uint32_t dstSel = dst_sel__mec_copy_data__tc_l2, + uint32_t dstCachePolicy = dst_cache_policy__mec_copy_data__stream, + uint32_t srcSel = src_sel__mec_copy_data__gpu_clock_count, + uint32_t srcCachePolicy = src_cache_policy__mec_copy_data__lru, + uint32_t countSel = count_sel__mec_copy_data__64_bits_of_data, + uint32_t wrConfirm = wr_confirm__mec_copy_data__wait_for_confirmation); + + size_t BuildBarrier( + void *pBuffer, + uint32_t eventIndex = event_index__mec_event_write__cs_partial_flush, + uint32_t eventType = CS_PARTIAL_FLUSH); + + size_t BuildAcquireMem( + uint8_t major, + void *pBuffer); + + size_t BuildScratch( + void *pScratchBase, + void *pBuffer); + + size_t BuildComputeShaderParams( + void *pBuffer); + + size_t BuildDispatch( + struct DispatchInfo *pInfo, + void *pBuffer); + + size_t BuildAtomicMem( + uint64_t *pAddr, + uint32_t atomic, + void *pBuffer, + uint32_t cachePolicy = cache_policy__mec_atomic_mem__stream, + uint64_t srcData = 1); +}; + +} // namespace core +} // namespace rocr + +#endif \ No newline at end of file diff --git a/inc/wddm/device.h b/inc/wddm/device.h new file mode 100644 index 0000000000..7d912be039 --- /dev/null +++ b/inc/wddm/device.h @@ -0,0 +1,245 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2020, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +#ifndef _ROCR_WDDM_H_ +#define _ROCR_WDDM_H_ + +#include +#include + +#include +#include +#include + +#include "inc/wddm/types.h" +#include "inc/rocr_proxy/rocr_proxy.h" +#include "inc/wddm/va_mgr.h" +#include "inc/wddm/status.h" +#include "inc/wddm/types.h" +#include "inc/wddm/gpu_memory.h" +#include "inc/wddm/cmd_util.h" + +namespace rocr { +namespace core { + +//class Queue; +class WDDMQueue; + +// WSL2 hyperv GPADL protocol limitation +#define MAX_USERPTR_BLOCK_SIZE 0xf0000000 + +class WDDMDevice { +public: + static constexpr size_t GpuMemoryChunkSize = 2 * (1ULL << 30); // 2 GB + + WDDMDevice(D3DKMT_HANDLE adapter, LUID adapter_luid); + ~WDDMDevice(); + + int Major() { return device_info_.major; } + int Minor() { return device_info_.minor; } + int Stepping() { return device_info_.stepping; } + bool IsDgpu() { return device_info_.is_dgpu; } + const char *ProductName() { return device_info_.product_name; } + const char *Uuid() { return device_info_.uuid; } + rocr_proxy::AsicFamilyType GfxFamily() { return device_info_.family; } + uint32_t DeviceId() { return device_info_.device_id; } + uint32_t WavefrontSize() { return device_info_.wavefront_size; } + uint32_t ComputeUnitCount() { return device_info_.compute_unit_count; } + uint32_t MaxEngineClockMhz() { return device_info_.max_engine_clock_mhz; } + uint32_t WatchPointsNum() { return device_info_.watch_points_num; } + uint32_t PciBusAddr() { return device_info_.pci_bus_addr; } + + uint32_t MemoryBusWidth() { return device_info_.memory_bus_width; } + uint32_t MaxMemoryClockMhz() { return device_info_.max_memory_clock_mhz; } + uint32_t WavePerCu() { return device_info_.wave_per_cu; } + uint32_t SimdPerCu() { return device_info_.simd_per_cu; } + uint32_t MaxScratchSlotsPerCu() { return device_info_.max_scratch_slots_per_cu; } + uint32_t NumShaderEngine() { return device_info_.num_shader_engine; } + uint32_t ShaderArrayPerShaderEngine() { return device_info_.shader_array_per_shader_engine; } + uint32_t NumSdmaEngine() { return device_info_.sdma_schedid.size(); } + uint32_t Domain() { return device_info_.domain; } + uint32_t NumGws() { return device_info_.num_gws; } + uint32_t AsicRevision() { return device_info_.asic_revision; } + uint64_t LocalHeapSize() { return device_info_.local_visible_heap_size + device_info_.local_invisible_heap_size; } + uint64_t LocalVisibleHeapSize() { return device_info_.local_visible_heap_size; } + uint64_t LocalInvisibleHeapSize() { return device_info_.local_invisible_heap_size; } + uint64_t PrivateApertureBase() { return device_info_.private_aperture_base; } + uint64_t PrivateApertureSize() { return device_info_.private_aperture_size; } + uint64_t SharedApertureBase() { return device_info_.shared_aperture_base; } + uint64_t SharedApertureSize() { return device_info_.shared_aperture_size; } + uint32_t LdsSize() { return device_info_.lds_size; } + uint64_t GPUCounterFrequency() { return device_info_.gpu_counter_frequency; } + uint32_t GetSwsQueueSize(void) const { return device_info_.user_queue_size; } + uint32_t GetMecFwVersion() { return device_info_.mec_fw_version; } + uint32_t GetSdmaFwVersion() { return device_info_.sdma_fw_version; } + uint32_t GetL1CacheSize() { return device_info_.l1_cache_size; } + uint32_t GetL2CacheSize() { return device_info_.l2_cache_size; } + uint32_t GetL3CacheSize() { return device_info_.l3_cache_size; } + uint32_t Gl2CacheLineSize() { return device_info_.gl2_cacheline_size; } + bool SupportStateShadowingByCpFw(void) const { return device_info_.state_shadowing_by_cpfw; } + bool SupportPlatformAtomic(void) const { return device_info_.platform_atomic_support; } + uint32_t GetSdmaEngine(uint32_t idx) { + assert(idx < NumSdmaEngine()); + return device_info_.sdma_schedid[idx]; + } + uint32_t GetComputeEngine() { return device_info_.compute_schedid; } + + uint64_t VramAvail(); + + void GetClockCounters(uint64_t *gpu, uint64_t *cpu); + uint32_t GetNumCpQueues() { return device_info_.num_cp_queues; } + + bool CreateSyncobj(D3DKMT_HANDLE *handle, uint64_t **addr); + void DestroySyncobj(D3DKMT_HANDLE handle); + + bool CreateQueue(WDDMQueue *queue); + void DestroyQueue(WDDMQueue *queue); + bool CreateHwQueue(WDDMQueue *queue); + bool DestroyHwQueue(WDDMQueue *queue); + bool SubmitToSwQueue(WDDMQueue *queue, uint64_t command_addr, + uint64_t command_size, uint64_t fence_value); + bool SubmitToHwQueue(WDDMQueue *queue, uint64_t command_addr, + uint64_t command_size, uint64_t fence_value); + + bool WaitPagingFence(WDDMQueue *queue) { + uint64_t value = page_fence_value_; + + if (*page_fence_addr_ < value && + !GpuWait(queue, &page_syncobj_, &value, 1)) + return false; + + return true; + } + + bool GpuWait(WDDMQueue *queue, const D3DKMT_HANDLE *syncobjs, + uint64_t *values, int count); + bool GpuSignal(D3DKMT_HANDLE context, const D3DKMT_HANDLE *syncobjs, + uint64_t *value, int count); + bool CpuWait(const D3DKMT_HANDLE *syncobjs, uint64_t *value, + int count, bool wait_any); + bool WaitOnPagingFenceFromCpu(); + + uint32_t LdsBlocks(const hsa_kernel_dispatch_packet_t *pkt); + uint32_t GetCmdbufSize(void) const { return cmdbuf_size_; } + uint32_t GetAqlFrameSize(void) const { return cmdbuf_aql_frame_size_; } + static uint32_t GetAqlFrameNum(void) { return cmdbuf_aql_frame_num_; } + + // Both legacy HWS and stage 1 HWS use KMD to alloc use queue memory, + // return false by default + bool AllocUserQueueMemFromUMD(void) const { return false; } + + bool IsHwsEnabled(int engine) { + return rocr_proxy::GetHwsEnabled(engine, &device_info_); + } + + void UpdatePageFence(uint64_t fence_value); + + D3DKMT_HANDLE PagingQueue() const { return page_queue_; } + D3DKMT_HANDLE PagingFence() const { return page_syncobj_; } + D3DKMT_HANDLE DeviceHandle() const { return device_; } + LUID GetLuid() const { return adapter_luid_; } + + const rocr_proxy::DeviceInfo& DeviceInfo() const { return device_info_; } + + ErrorCode ReserveGpuVirtualAddress(rocr_proxy::AllocDomain domain, + gpusize hit_base_addr, + gpusize size, + gpusize *out_gpu_virtual_addr, + gpusize alignment, + bool lock=false); + + ErrorCode FreeGpuVirtualAddress(rocr_proxy::AllocDomain domain, + gpusize base_addr, + gpusize size); + + ErrorCode CreateGpuMemory(const GpuMemoryCreateInfo &create_info, GpuMemory **gpu_mem); +private: + bool ParseDeviceInfo(void); + void DestroyDeviceInfo(void); + bool CreateDevice(void); + bool DestroyDevice(void); + bool CreatePagingQueue(void); + bool DestroyPagingQueue(void); + void *Lock(D3DKMT_HANDLE handle); + bool Unlock(D3DKMT_HANDLE handle); + bool CreateContext(int engine, D3DKMT_HANDLE *handle); + bool DestroyContext(D3DKMT_HANDLE handle); + + void SetPowerOptimization(bool restore); + void InitCmdbufInfo(void); + bool ReserveSystemHeapSpace(void); + bool FreeSystemHeapSpace(void); + bool ReserveLocalHeapSpace(void); + bool CommitSystemHeapSpace(void* addr, int64_t size, bool lock=false); + bool DecommitSystemHeapSpace(void* addr, int64_t size); + bool FreeLocalHeapSpace(void); + void InitVaMgr(); + + D3DKMT_HANDLE adapter_; + LUID adapter_luid_; + D3DKMT_HANDLE device_; + + D3DKMT_HANDLE page_queue_; + D3DKMT_HANDLE page_syncobj_; + uint64_t *page_fence_addr_; + std::atomic page_fence_value_; + + uint64_t local_heap_space_start_; + uint64_t local_heap_space_size_; + uint64_t system_heap_space_start_; + uint64_t system_heap_space_size_; + uint32_t cmdbuf_size_; + uint32_t cmdbuf_aql_frame_size_; + static const uint32_t cmdbuf_aql_frame_num_; + // device info + rocr_proxy::DeviceInfo device_info_; + + std::unique_ptr local_va_mgr_; + //CmdUtil cmd_util; +}; + +NTSTATUS WDDMGetAdapters(D3DKMT_ADAPTERINFO *&adapters, int &num_adapters); + +} // namespace core +} // namespace rocr + +#endif diff --git a/inc/wddm/gpu_memory.h b/inc/wddm/gpu_memory.h new file mode 100644 index 0000000000..b04a5d85d3 --- /dev/null +++ b/inc/wddm/gpu_memory.h @@ -0,0 +1,218 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2020, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +#ifndef _ROCR_GPU_MEMORY_H_ +#define _ROCR_GPU_MEMORY_H_ + +#include +#include +#include "util/utils.h" +#include "inc/wddm/types.h" +#include "inc/wddm/thunks.h" +#include "inc/rocr_proxy/rocr_proxy.h" + +namespace rocr { +namespace core { + +class WDDMDevice; + +union GpuMemoryCreateFlags { + struct { + uint64_t virtual_alloc : 1; + uint64_t physical_only : 1; + uint64_t interprocess : 1; + uint64_t locked : 1; + uint64_t unused : 60; + }; + uint64_t reserved; +}; + +struct GpuMemoryCreateInfo { + GpuMemoryCreateInfo() { + flags.reserved = 0; + domain = rocr_proxy::kLocal; + size = 0; + alignment = 0; + mem_flags = 0; + engine_flag = 0; + va_hint = 0; + user_ptr = nullptr; + dmabuf_fd = -1; + } + + GpuMemoryCreateFlags flags; + rocr_proxy::AllocDomain domain; + gpusize size; + gpusize alignment; + int mem_flags; + int engine_flag; + int dmabuf_fd; // Import from dmabuf + + void *user_ptr; + gpusize va_hint; +}; + +struct GpuMemoryDesc { + GpuMemoryDesc() { + gpu_addr = 0; + cpu_addr = nullptr; + client_size = 0; + size = alignment = 0; + flags.reserved = 0; + mem_flags = 0; + engine_flag = 0; + } + + rocr_proxy::AllocDomain domain; + LUID adapter_luid; // Where is the backing store location + gpusize gpu_addr; + void *cpu_addr; + gpusize client_size; // user request size + gpusize size; + gpusize alignment; + + union { + struct { + uint32_t is_virtual : 1; + uint32_t is_shared : 1; + uint32_t is_external : 1; + uint32_t is_physical_only : 1; + uint32_t is_locked : 1; + + uint32_t unused : 27; + }; + + uint32_t reserved; + } flags; + + int mem_flags; + int engine_flag; +}; + +struct SharedHandleInfo { + rocr_proxy::AllocDomain domain; + LUID adapter_luid; + gpusize client_size; // user request size + uint64_t size; + uint32_t flags; + int mem_flags; +}; + +using GpuMemoryHandle = void *; + +class GpuMemory { +public: + static size_t CalcChunkNumbers(gpusize size); + + ErrorCode Init(const GpuMemoryCreateInfo &create_info); + + WDDMDevice *GetDevice() const { return device_; } + gpusize Size() const { return desc_.size; } + gpusize ClientSize() const { return desc_.client_size; } + uint64_t GpuAddress() const { return desc_.gpu_addr; } + void *CpuAddress() const { return desc_.cpu_addr; } + + inline bool IsLocal() const { return desc_.domain == rocr_proxy::kLocal; } + inline bool IsUserMemory() const { return desc_.domain == rocr_proxy::kUserMemory; } + inline bool IsSystem() const { return desc_.domain == rocr_proxy::kSystem; } + inline bool IsUserQueue() const { return desc_.domain == rocr_proxy::kUserQueue; } + inline bool IsPhysicalOnly() const { return desc_.flags.is_physical_only; } + inline bool IsVirtual() const { return desc_.flags.is_virtual; } + inline bool IsShared() const { return desc_.flags.is_shared; } + inline bool IsExternal() const { return desc_.flags.is_external; } + + inline uint32_t Flags() const { return desc_.flags.reserved; } + inline int GetAllocInfo() const { return desc_.mem_flags; } + inline bool IsFineGrain() const { return (desc_.mem_flags & rocr_proxy::kFineGrain); } + inline bool IsSameAdapter(const LUID &luid) const { + return (desc_.adapter_luid.HighPart == luid.HighPart && + desc_.adapter_luid.LowPart == luid.LowPart); + } + + WinAllocationHandle GetAllocationHandle(size_t index) const { return alloc_handles_ptr_[index]; } + size_t NumChunks() const { return num_allocations_; } + + const GpuMemoryHandle GetGpuMemoryHandle() const { + return reinterpret_cast(const_cast(this)); + } + + static GpuMemory *Convert(GpuMemoryHandle handle) { return reinterpret_cast(handle); } + + ErrorCode ReserveGpuVirtualAddress(gpusize base_virt_addr, gpusize va_size, gpusize alignment); + ErrorCode FreeGpuVirtualAddress(gpusize va_start_address, gpusize va_size); + + ErrorCode MapGpuVirtualAddress(const gpusize map_addr, const gpusize size, gpusize offset = 0); + ErrorCode UnmapGpuVirtualAddress(const gpusize map_addr, const gpusize size, gpusize offset = 0); + + ErrorCode MakeResident(); + ErrorCode Evict(); + + ErrorCode ExportPhysicalHandle(int* dmabuf_fd, uint32_t flags = SHARED_ALLOCATION_ALL_ACCESS); + ErrorCode ImportPhysicalHandle(int dmabuf_fd); + ~GpuMemory(); +protected: + explicit GpuMemory(WDDMDevice *device); +private: + ErrorCode CreatePhysicalMemory(); + ErrorCode FreePhysicalMemory(); + + uint64_t AdjustSize(gpusize size) const; +private: + friend class WDDMDevice; + + WDDMDevice *const device_; + + GpuMemoryDesc desc_; + + size_t num_allocations_; + WinAllocationHandle *alloc_handles_ptr_; + WinAllocationHandle alloc_handle_; // Optimization for num_allocations_ is 1 + + WinResourceHandle resource_; // Handle to a resource object that wraps the allocation. Used for shared resources + + DISALLOW_COPY_AND_ASSIGN(GpuMemory); +}; + +} // namespace core +} // namespace rocr + +#endif diff --git a/inc/wddm/queue.h b/inc/wddm/queue.h new file mode 100644 index 0000000000..51c9dfd453 --- /dev/null +++ b/inc/wddm/queue.h @@ -0,0 +1,284 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2020, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// +#ifndef _WDDM_QUEUE_H_ +#define _WDDM_QUEUE_H_ + +#include +#include +#include "inc/wddm/types.h" +#include "inc/wddm/device.h" +#include "inc/wddm/gpu_memory.h" +#include "hsa-runtime/inc/hsa_ext_amd.h" +#include "hsa-runtime/inc/amd_hsa_queue.h" +#include "hsa-runtime/inc/amd_hsa_signal.h" +#include "inc/wddm/cmd_util.h" + +namespace rocr { +namespace core { + +class Queue; +class WDDMDevice; + +class WDDMQueue { +public: + WDDMQueue(WDDMDevice *device, + uint32_t cmdbuf_size, + uint32_t engine, + bool use_hws = true) : + device(device), + context(NULL), + queue(NULL), + syncobj(NULL), + sync_addr(NULL), + cmdbuf(0), + cmdbuf_addr(0), + cmdbuf_size(cmdbuf_size), + queue_engine(engine), + use_hws(use_hws), + prio(rocr_proxy::kNormal) { + + } + + virtual ~WDDMQueue() { } + + virtual hsa_status_t Init(void) = 0; + virtual hsa_status_t Fini(void) = 0; + + hsa_status_t SwsInit(void); + hsa_status_t SwsFini(void); + hsa_status_t SwsSubmit(uint64_t command_addr, + uint64_t command_size, + uint64_t fence_value); + + hsa_status_t HwsInit(void); + hsa_status_t HwsFini(void); + hsa_status_t HwsSubmit(uint64_t command_addr, + uint64_t command_size, + uint64_t fence_value); + hsa_status_t SetPriority(hsa_amd_queue_priority_t priority); + + uint64_t *GetSyncAddr(void) const { return sync_addr; } + uint64_t GetCmdbufAddr(void) const { return cmdbuf_addr; } + + rocr_proxy::SchedLevel ConvertSchedLevel(hsa_amd_queue_priority_t prio) const { + switch (prio) { + case HSA_AMD_QUEUE_PRIORITY_LOW: + return rocr_proxy::kLow; + case HSA_AMD_QUEUE_PRIORITY_HIGH: + return rocr_proxy::kHigh; + case HSA_AMD_QUEUE_PRIORITY_NORMAL: + default: + return rocr_proxy::kNormal; + } + } + + WDDMDevice *device; + + D3DKMT_HANDLE context; + D3DKMT_HANDLE queue; + + D3DKMT_HANDLE syncobj; + uint64_t *sync_addr; + + GpuMemoryHandle cmdbuf; + uint64_t cmdbuf_addr; + uint32_t cmdbuf_size; + + GpuMemoryHandle queue_mem; + uint64_t queue_addr; + + uint32_t queue_engine; + + bool use_hws; + rocr_proxy::SchedLevel prio; +}; + +class ComputeQueue : public WDDMQueue { +public: + ComputeQueue(WDDMDevice *device, + void *ring, + uint64_t ring_size, + std::atomic *ring_wptr, + std::atomic *ring_rptr, + volatile int64_t *error_addr, + uint32_t cmdbuf_size, + uint32_t engine, + bool use_hws = true); + + ~ComputeQueue(); + + virtual hsa_status_t Init(void); + virtual hsa_status_t Fini(void); + virtual hsa_status_t Submit(void); + + void* GetRing(void) const { return ring; } + uint64_t GetRingSize(void) const { return ring_size; } + std::atomic* GetRingWptr(void) const { return ring_wptr; } + std::atomic* GetRingRptr(void) const { return ring_rptr; } + + uint64_t GetAqlWriteIndex(void) const { return cmdbuf_aql_frame_write_index; } + uint32_t GetAqlFrameSize(void) const { return cmdbuf_aql_frame_size; } + + bool IsInvalidPacket(void) const { + uint16_t *packet = (uint16_t *)((char *)ring + + (cmdbuf_aql_frame_write_index % ring_size) * 64); + return ((*packet >> HSA_PACKET_HEADER_TYPE) & ((1 << HSA_PACKET_HEADER_WIDTH_TYPE) - 1)) + == HSA_PACKET_TYPE_INVALID; + } + + hsa_status_t Process(void); + uint64_t * GetDoorbellPtr() const { return (uint64_t *)&doorbell_signal_.value; } + void RingDoorbell(); +private: + hsa_status_t KernelDispatchAqlToPm4(char *cpu, hsa_kernel_dispatch_packet_t *packet); + hsa_status_t BarrierGenericAqlToPm4(char *cpu, hsa_barrier_and_packet_t *packet, bool is_or = false); + struct amd_aql_pm4_ib { + uint16_t header; + uint16_t ven_hdr; + uint32_t ib_jump_cmd[4]; + uint32_t dw_cnt_remain; + uint32_t reserved[8]; + hsa_signal_t completion_signal; + }; + hsa_status_t VendorSpecificAqlToPm4(char *cpu, amd_aql_pm4_ib *packet); + hsa_status_t SwitchAql2PM4(void); + + hsa_status_t PreSubmit(void); + hsa_status_t EndSubmit(void); + + void *ring; + uint64_t ring_size; + std::atomic *ring_wptr; + std::atomic *ring_rptr; + + // ib_start_addr is the current ib start address + uint64_t ib_start_addr; + + // ib_size is the current ib size. + uint64_t ib_size; + + // record the last submitted aql frame write index + uint64_t sync_point; + + uint64_t cmdbuf_aql_frame_write_index; + uint32_t cmdbuf_aql_frame_size; + + bool needs_barrier; + bool ready_to_submit; + + CmdUtil cmd_util; + +private: + bool EnableProfiling() { + return AMD_HSA_BITS_GET(amd_queue_rocr_->queue_properties, AMD_QUEUE_PROPERTIES_ENABLE_PROFILING); + } + void HandleError(hsa_status_t status); + bool UpdateScratch(uint32_t private_segment_size, bool wave32); + + uint32_t UpdateIndexStride(uint32_t srd, bool wave32); + + void *ScratchBase() { return scratch_base_; } + + void AppendCmdbufSratchBaseOffset(int offset) { + scratch_base_offset_array_.push_back(offset); + } + + bool RelocateCmdbufScratchBase(uint64_t addr); + + uint32_t ScratchSizePerWave() { return scratch_size_per_wave_; } + uint64_t GetKernelObjAddr(uint64_t addr) const; + void InitScratchSRD(); + GpuMemoryHandle amd_queue_mem_; + amd_queue_t *amd_queue_; + amd_queue_t *amd_queue_rocr_; + amd_signal_t doorbell_signal_; + volatile std::atomic *error_code_; + std::thread aql_to_pm4_thread_; + bool thread_stop_; + std::mutex thread_cond_lock_; + std::condition_variable thread_cond_; + static void AqlToPm4Thread(ComputeQueue *queue); + + uint32_t scratch_waves_; + uint32_t scratch_size_per_wave_; + uint32_t scratch_size_; + void *scratch_base_; + GpuMemoryHandle scratch_mem_; + + std::vector scratch_base_offset_array_; +}; + +class SDMAQueue : public WDDMQueue { +public: + SDMAQueue(WDDMDevice *device, + uint64_t cmdbuf_size, + uint32_t engine, + bool use_hws = true) : + WDDMQueue(device, cmdbuf_size, engine, use_hws), + rptr_next(0), + ib_size(0), + ib_start_addr(0) { + + } + + virtual ~SDMAQueue() { } + + hsa_status_t Init(void); + hsa_status_t Fini(void); + hsa_status_t Submit(void); + + int PreparePacket(uint32_t offset, uint64_t size); + + void WaitQueue(void) { + device->CpuWait(&syncobj, &rptr_next, 1, false); + } + +private: + uint64_t rptr_next; + uint64_t ib_size; + uint64_t ib_start_addr; +}; + +} +} + +#endif diff --git a/inc/wddm/status.h b/inc/wddm/status.h new file mode 100644 index 0000000000..96808622ef --- /dev/null +++ b/inc/wddm/status.h @@ -0,0 +1,59 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2020, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +#ifndef _ROCR_CORE_INC_WDDM_STATUS_H_ +#define _ROCR_CORE_INC_WDDM_STATUS_H_ + +enum class ErrorCode { + Success, + DeviceLost, + UnSupported, + NotReady, + OutOfMemory, + OutOfGpuMemory, + Timeout, + SyscallFail, + InvalidateParams, + Unknown, +}; + +#endif diff --git a/inc/wddm/thunks.h b/inc/wddm/thunks.h new file mode 100644 index 0000000000..15556a8ab5 --- /dev/null +++ b/inc/wddm/thunks.h @@ -0,0 +1,232 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2020, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +#ifndef _ROCR_CORE_INC_WDDM_THUNKS_H_ +#define _ROCR_CORE_INC_WDDM_THUNKS_H_ + +#include "inc/wddm/status.h" +#include "inc/wddm/types.h" + +namespace rocr { +namespace core { + +inline ErrorCode TranslateNtStatus(NTSTATUS status) { + switch (status) { + case STATUS_SUCCESS: + return ErrorCode::Success; + case STATUS_PENDING: + return ErrorCode::NotReady; + case STATUS_NO_MEMORY: + return ErrorCode::OutOfMemory; + case STATUS_DEVICE_REMOVED: + return ErrorCode::DeviceLost; + case STATUS_GRAPHICS_NO_VIDEO_MEMORY: + return ErrorCode::OutOfGpuMemory; + case STATUS_TIMEOUT: + return ErrorCode::Timeout; + case STATUS_INVALID_PARAMETER: + return ErrorCode::InvalidateParams; + default: + break; + } + return ErrorCode::Unknown; +} + +namespace thunk { + +typedef D3DKMT_CREATEALLOCATION CreateAllocationArgs; +typedef D3DKMT_CREATECONTEXT CreateContextArgs; +typedef D3DKMT_CREATECONTEXTVIRTUAL CreateContextVirtualArgs; +typedef D3DKMT_CREATEPAGINGQUEUE CreatePagingQueueArgs; +typedef D3DKMT_CREATESYNCHRONIZATIONOBJECT CreateSynchronizationObjectArgs; +typedef D3DKMT_CREATESYNCHRONIZATIONOBJECT2 CreateSynchronizationObject2Args; +typedef D3DKMT_ESCAPE EscapeArgs; +typedef D3DKMT_EVICT EvictArgs; +typedef D3DKMT_FREEGPUVIRTUALADDRESS FreeGpuVirtualAddressArgs; +typedef D3DKMT_LOCK LockArgs; +typedef D3DKMT_LOCK2 Lock2Args; +typedef D3DKMT_OPENRESOURCE OpenResourceArgs; +typedef D3DKMT_OPENRESOURCEFROMNTHANDLE OpenResourceFromNtHandleArgs; +typedef D3DKMT_QUERYADAPTERINFO QueryAdapterInfoArgs; +typedef D3DKMT_SIGNALSYNCHRONIZATIONOBJECT SignalSynchronizationObjectArgs; +typedef D3DKMT_SIGNALSYNCHRONIZATIONOBJECT2 SignalSynchronizationObject2Args; +typedef D3DKMT_SIGNALSYNCHRONIZATIONOBJECTFROMCPU SignalSynchronizationObjectFromCpuArgs; +typedef D3DKMT_SIGNALSYNCHRONIZATIONOBJECTFROMGPU2 SignalSynchronizationObjectFromGpuArgs; +typedef D3DKMT_SUBMITCOMMAND SubmitCommandArgs; +typedef D3DKMT_UNLOCK UnlockArgs; +typedef D3DKMT_UNLOCK2 Unlock2Args; +typedef D3DKMT_UPDATEGPUVIRTUALADDRESS UpdateGpuVirtualAddressArgs; +typedef D3DKMT_WAITFORSYNCHRONIZATIONOBJECT WaitForSynchronizationObjectArgs; +typedef D3DKMT_WAITFORSYNCHRONIZATIONOBJECT2 WaitForSynchronizationObject2Args; +typedef D3DKMT_WAITFORSYNCHRONIZATIONOBJECTFROMCPU WaitForSynchronizationObjectFromCpuArgs; +typedef D3DKMT_WAITFORSYNCHRONIZATIONOBJECTFROMGPU WaitForSynchronizationObjectFromGpuArgs; +typedef D3DKMT_ACQUIREKEYEDMUTEX AcquireKeyedMutexArgs; +typedef D3DKMT_RELEASEKEYEDMUTEX ReleaseKeyedMutexArgs; +typedef D3DKMT_OPENKEYEDMUTEX OpenKeyedMutexArgs; +typedef D3DKMT_DESTROYKEYEDMUTEX DestroyKeyedMutexArgs; +typedef D3DKMT_QUERYVIDEOMEMORYINFO QueryVideoMemoryInfoArgs; +typedef D3DKMT_CREATEHWQUEUE CreateHwQueueArgs; +typedef D3DKMT_DESTROYHWQUEUE DestroyHwQueueArgs; +typedef D3DKMT_SUBMITCOMMANDTOHWQUEUE SubmitCommandToHwQueueArgs; +typedef D3DKMT_SUBMITPRESENTTOHWQUEUE SubmitPresentToHwQueueArgs; +typedef D3DKMT_SUBMITSIGNALSYNCOBJECTSTOHWQUEUE SubmitSignalSyncObjectsToHwQueueArgs; +typedef D3DKMT_SUBMITWAITFORSYNCOBJECTSTOHWQUEUE SubmitWaitForSyncObjectsToHwQueueArgs; +typedef D3DKMT_CREATESYNCFILE CreateSyncFileArgs; + +inline ErrorCode MapGpuVirtualAddress(D3DDDI_MAPGPUVIRTUALADDRESS *args) { + return TranslateNtStatus(D3DKMTMapGpuVirtualAddress(args)); +} + +inline ErrorCode CreateAllocation(CreateAllocationArgs *args) { + return TranslateNtStatus(D3DKMTCreateAllocation2(args)); +} + +inline ErrorCode DestroyAllocation( + WinDeviceHandle device, + WinResourceHandle resource, + size_t num_allocations, + const WinAllocationHandle *alloc_handles) { + + D3DKMT_DESTROYALLOCATION2 args{}; + + memset(&args, 0, sizeof(args)); + args.hDevice = device; + if (resource) { + args.hResource = resource; + } else { + args.phAllocationList = alloc_handles; + args.AllocationCount = num_allocations; + } + + return TranslateNtStatus(D3DKMTDestroyAllocation2(&args)); +} + +inline ErrorCode ReserveGpuVirtualAddress(D3DDDI_RESERVEGPUVIRTUALADDRESS *args) { + return TranslateNtStatus(D3DKMTReserveGpuVirtualAddress(args)); +} + +inline ErrorCode ReserveGpuVirtualAddress(WinAdapterHandle handle, + gpusize size, + gpusize base_address, + gpusize *out_addr) { + D3DDDI_RESERVEGPUVIRTUALADDRESS args{}; + args.hPagingQueue = handle; + args.Size = size; + args.BaseAddress = base_address; + + auto code = ReserveGpuVirtualAddress(&args); + if (code == ErrorCode::Success) + *out_addr = args.VirtualAddress; + return code; +} + +inline ErrorCode ReserveGpuVirtualAddress(WinAdapterHandle handle, + gpusize size, + gpusize minimum_address, + gpusize maximum_address, + gpusize *out_addr) { + D3DDDI_RESERVEGPUVIRTUALADDRESS args{}; + args.hPagingQueue = handle; + args.Size = size; + args.MinimumAddress = minimum_address; + args.MaximumAddress = maximum_address; + + auto code = ReserveGpuVirtualAddress(&args); + if (code == ErrorCode::Success) + *out_addr = args.VirtualAddress; + return code; +} + +inline ErrorCode FreeGpuVirtualAddress(FreeGpuVirtualAddressArgs *args) { + return TranslateNtStatus(D3DKMTFreeGpuVirtualAddress(args)); +} + +inline ErrorCode FreeGpuVirtualAddress(WinAdapterHandle handle, + gpusize base_address, + gpusize size) { + FreeGpuVirtualAddressArgs args{}; + args.hAdapter = handle; + args.Size = size; + args.BaseAddress = base_address; + return FreeGpuVirtualAddress(&args); +} + +inline ErrorCode MakeResident(D3DDDI_MAKERESIDENT *args) { + return TranslateNtStatus(D3DKMTMakeResident(args)); +} + +inline ErrorCode Evict(EvictArgs *args) { + return TranslateNtStatus(D3DKMTEvict(args)); +} + +inline ErrorCode ShareObjects(size_t num_allocations, + WinResourceHandle resource, + uint32_t flags, + int* dmabuf_fd) { + OBJECT_ATTRIBUTES obj_attr; + HANDLE nt_handle; + ErrorCode ret; + + InitializeObjectAttributes(&obj_attr, nullptr, OBJ_INHERIT, nullptr, nullptr); + ret = TranslateNtStatus(D3DKMTShareObjects(num_allocations, + &resource, &obj_attr, flags, &nt_handle)); + if (ret == ErrorCode::Success) + *dmabuf_fd = *(reinterpret_cast(&nt_handle)); + else + *dmabuf_fd = -1; + + return ret; +} + +inline ErrorCode QueryResourceInfoFromNtHandle(D3DKMT_QUERYRESOURCEINFOFROMNTHANDLE *args) { + return TranslateNtStatus(D3DKMTQueryResourceInfoFromNtHandle(args)); +} + +inline ErrorCode OpenResourceFromNtHandle(D3DKMT_OPENRESOURCEFROMNTHANDLE *args) { + return TranslateNtStatus(D3DKMTOpenResourceFromNtHandle(args)); +} + +} // namespace thunk +} // namespace core +} // namespace rocr + +#endif // _ROCR_CORE_INC_WDDM_THUNKS_H_ diff --git a/inc/wddm/types.h b/inc/wddm/types.h new file mode 100644 index 0000000000..cd831158ce --- /dev/null +++ b/inc/wddm/types.h @@ -0,0 +1,101 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2020, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +#ifndef _ROCR_CORE_INC_WDDM_TYPES_H_ +#define _ROCR_CORE_INC_WDDM_TYPES_H_ + +#include +#include +#include "inc/rocr_proxy/wddm_types.h" +// windows wchar is 16bit, but linux is 32bit +// seems libdxcore (not dxgkrnl.ko) convert thunk windows wchar to linux one +// so only accept 32bit wchar args. note driver private data structure still +// use 16bit wchar +#define WCHAR wchar_t +#define PCWSTR const wchar_t * +#include +#undef WCHAR +#undef PCWSTR + +using gpusize = uint64_t; // Used to specify GPU addresses and sizes of GPU allocations +using WinAllocationHandle = D3DKMT_HANDLE; +using WinResourceHandle = D3DKMT_HANDLE; +using WinContextHandle = D3DKMT_HANDLE; +using WinDeviceHandle = D3DKMT_HANDLE; +using WinAdapterHandle = D3DKMT_HANDLE; + +//reference dk/winnt.h +#define STANDARD_RIGHTS_REQUIRED (0x000F0000L) + +//reference dk/ntdef.h +#define OBJ_INHERIT (0x00000002L) +typedef WCHAR *PWCHAR, *LPWCH, *PWCH; +typedef struct _UNICODE_STRING { + USHORT Length; + USHORT MaximumLength; +#ifdef MIDL_PASS + [size_is(MaximumLength / 2), length_is((Length) / 2) ] USHORT * Buffer; +#else // MIDL_PASS + _Field_size_bytes_part_opt_(MaximumLength, Length) PWCH Buffer; +#endif // MIDL_PASS +} UNICODE_STRING; +typedef UNICODE_STRING *PUNICODE_STRING; +typedef const UNICODE_STRING *PCUNICODE_STRING; + +typedef struct _OBJECT_ATTRIBUTES { + ULONG Length; + HANDLE RootDirectory; + PUNICODE_STRING ObjectName; + ULONG Attributes; + PVOID SecurityDescriptor; + PVOID SecurityQualityOfService; +} OBJECT_ATTRIBUTES; +#define InitializeObjectAttributes( p, n, a, r, s ) { \ + (p)->Length = sizeof( OBJECT_ATTRIBUTES ); \ + (p)->RootDirectory = r; \ + (p)->Attributes = a; \ + (p)->ObjectName = n; \ + (p)->SecurityDescriptor = s; \ + (p)->SecurityQualityOfService = NULL; \ + } + +#endif // _ROCR_CORE_INC_WDDM_TYPES_H_ diff --git a/inc/wddm/va_mgr.h b/inc/wddm/va_mgr.h new file mode 100644 index 0000000000..12dac08c67 --- /dev/null +++ b/inc/wddm/va_mgr.h @@ -0,0 +1,86 @@ +#ifndef VA_MGR_H_ +#define VA_MGR_H_ + +#include +#include +#include "util/utils.h" + +namespace rocr { +namespace core { + +class VaMgr { +public: + VaMgr(uint64_t start, uint64_t size, uint64_t min_align); + ~VaMgr(); + + /* Allocate `bytes` VA, if `align` is not zero, the returned address is aligned by `align`. + * If `addr` parameter is not zero, try best to allocate VA from fixed address `addr`. + */ + uint64_t Alloc(uint64_t bytes, uint64_t align, uint64_t addr = 0); + + void Free(uint64_t addr); + +private: + uint64_t AllocImpl(uint64_t bytes, uint64_t align); + + struct Fragment { + using ptr = std::multimap::iterator; + ptr free_list_entry_; + + struct { + uint64_t size : 63; + bool is_free : 1; + }; + + Fragment() : size(0), is_free(false) {} + Fragment(ptr iterator, uint64_t len, bool is_free) + : free_list_entry_(iterator), size(len), is_free(is_free) {} + }; + + static inline Fragment make_fragment(typename Fragment::ptr iter, uint64_t len) { + return {iter, len, true}; + } + + inline Fragment make_fragment(uint64_t len) { return {free_list_.end(), len, false}; } + + static inline bool is_free(const Fragment& f) { return f.is_free; } + void set_used(Fragment& f) { + f.is_free = false; + f.free_list_entry_ = free_list_.end(); + } + static void set_free(Fragment& f, typename Fragment::ptr iter) { + f.free_list_entry_ = iter; + f.is_free = true; + } + + inline void remove_free_list_entry(Fragment& frag) { + if (frag.free_list_entry_ != free_list_.end()) { + free_list_.erase(frag.free_list_entry_); + frag.free_list_entry_ = free_list_.end(); + } + } + + inline void add_free_fragment(uint64_t size, uint64_t base) { + auto it = free_list_.insert(std::make_pair(size, base)); + frag_map_[base] = make_fragment(it, size); + } + + inline void add_used_fragment(uint64_t size, uint64_t base) { + frag_map_[base] = make_fragment(size); + } + // Indexed by size + std::multimap free_list_; + // Indexed by VA, each fragment has no overlap + std::map frag_map_; + + uint64_t min_align_; + + std::mutex lock_; // Mutex protecting allocation and free of va + + + DISALLOW_COPY_AND_ASSIGN(VaMgr); +}; + +} // namespace core +} // namespace rocr +#endif // VA_MGR_H_ diff --git a/libdrm.cpp b/libdrm.cpp new file mode 100644 index 0000000000..2e2cb8aeac --- /dev/null +++ b/libdrm.cpp @@ -0,0 +1,69 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2020, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// +#include +#include +#include + +#include "inc/wddm/types.h" +#include "inc/wddm/device.h" +#include "libhsakmt.h" + +HSAKMT_STATUS HSAKMTAPI hsaKmtGetAMDGPUDeviceHandle( + HSAuint32 NodeId, HsaAMDGPUDeviceHandle *DeviceHandle) { + CHECK_DXG_OPEN(); + + rocr::core::WDDMDevice *pDevice = get_wddmdev(NodeId); + if (pDevice != nullptr) { + *DeviceHandle = reinterpret_cast(pDevice); + return HSAKMT_STATUS_SUCCESS; + } + return HSAKMT_STATUS_ERROR; +} + +HSAKMTAPI int hsaKmtamdgpu_query_gpu_info(void *dev, + struct amdgpu_gpu_info *info) { + rocr::core::WDDMDevice *pDevice = + reinterpret_cast(dev); + memset(info, 0, sizeof(*info)); + info->gpu_counter_freq = pDevice->GPUCounterFrequency() / 1000ull; + return 0; +} diff --git a/libhsakmt.h b/libhsakmt.h new file mode 100644 index 0000000000..471c056307 --- /dev/null +++ b/libhsakmt.h @@ -0,0 +1,158 @@ +/* + * Copyright © 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef LIBHSAKMT_H_INCLUDED +#define LIBHSAKMT_H_INCLUDED + +#include +#include +#include +#include "hsakmt/hsakmt.h" + +#include "inc/wddm/types.h" +#include "inc/wddm/device.h" + +rocr::core::WDDMDevice* get_wddmdev(uint32_t node_id); + +extern unsigned long dxg_open_count; +extern bool hsakmt_forked; +extern pthread_mutex_t hsakmt_mutex; +extern bool is_dgpu; +extern bool is_svm_api_supported; +extern int zfb_support; +extern int vendor_packet_support; + +#undef HSAKMTAPI +#define HSAKMTAPI __attribute__((visibility ("default"))) + +#if defined(__clang__) +#if __has_feature(address_sanitizer) +#define SANITIZER_AMDGPU 1 +#endif +#endif + +/*Avoid pointer-to-int-cast warning*/ +#define PORT_VPTR_TO_UINT64(vptr) ((uint64_t)(unsigned long)(vptr)) + +/*Avoid int-to-pointer-cast warning*/ +#define PORT_UINT64_TO_VPTR(v) ((void*)(unsigned long)(v)) + +#define CHECK_DXG_OPEN() \ + do { if (dxg_open_count == 0 || hsakmt_forked) return HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED; } while (0) + +/* Might be defined in limits.h on platforms where it is constant (used by musl) */ +/* See also: https://pubs.opengroup.org/onlinepubs/7908799/xsh/limits.h.html */ +#ifndef PAGE_SIZE +extern int PAGE_SIZE; +#endif +extern int PAGE_SHIFT; + +/* 64KB BigK fragment size for TLB efficiency */ +#define GPU_BIGK_PAGE_SIZE (1 << 16) + +/* 2MB huge page size for 4-level page tables on Vega10 and later GPUs */ +#define GPU_HUGE_PAGE_SIZE (2 << 20) + +#define CHECK_PAGE_MULTIPLE(x) \ + do { if ((uint64_t)PORT_VPTR_TO_UINT64(x) % PAGE_SIZE) return HSAKMT_STATUS_INVALID_PARAMETER; } while(0) + +#define ALIGN_UP(x,align) (((uint64_t)(x) + (align) - 1) & ~(uint64_t)((align)-1)) +#define ALIGN_UP_32(x,align) (((uint32_t)(x) + (align) - 1) & ~(uint32_t)((align)-1)) +#define PAGE_ALIGN_UP(x) ALIGN_UP(x,PAGE_SIZE) +#define BITMASK(n) ((n) ? (UINT64_MAX >> (sizeof(UINT64_MAX) * CHAR_BIT - (n))) : 0) +#define ARRAY_LEN(array) (sizeof(array) / sizeof(array[0])) + +/* HSA Thunk logging usage */ +extern int hsakmt_debug_level; +#define hsakmt_print(level, fmt, ...) \ + do { if (level <= hsakmt_debug_level) fprintf(stderr, fmt, ##__VA_ARGS__); } while (0) +#define HSAKMT_DEBUG_LEVEL_DEFAULT -1 +#define HSAKMT_DEBUG_LEVEL_ERR 3 +#define HSAKMT_DEBUG_LEVEL_WARNING 4 +#define HSAKMT_DEBUG_LEVEL_INFO 6 +#define HSAKMT_DEBUG_LEVEL_DEBUG 7 +#define pr_err(fmt, ...) \ + hsakmt_print(HSAKMT_DEBUG_LEVEL_ERR, fmt, ##__VA_ARGS__) +#define pr_warn(fmt, ...) \ + hsakmt_print(HSAKMT_DEBUG_LEVEL_WARNING, fmt, ##__VA_ARGS__) +#define pr_info(fmt, ...) \ + hsakmt_print(HSAKMT_DEBUG_LEVEL_INFO, fmt, ##__VA_ARGS__) +#define pr_debug(fmt, ...) \ + hsakmt_print(HSAKMT_DEBUG_LEVEL_DEBUG, fmt, ##__VA_ARGS__) +#define pr_err_once(fmt, ...) \ +({ \ + static bool __print_once; \ + if (!__print_once) { \ + __print_once = true; \ + pr_err(fmt, ##__VA_ARGS__); \ + } \ +}) +#define pr_warn_once(fmt, ...) \ +({ \ + static bool __print_once; \ + if (!__print_once) { \ + __print_once = true; \ + pr_warn(fmt, ##__VA_ARGS__); \ + } \ +}) + +/* Expects HSA_ENGINE_ID.ui32, returns gfxv (full) in hex */ +#define HSA_GET_GFX_VERSION_FULL(ui32) \ + (((ui32.Major) << 16) | ((ui32.Minor) << 8) | (ui32.Stepping)) + +HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id); +HSAKMT_STATUS gpuid_to_nodeid(uint32_t gpu_id, uint32_t* node_id); +bool prefer_ats(HSAuint32 node_id); +uint16_t get_device_id_by_node_id(HSAuint32 node_id); +uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id); +uint32_t get_direct_link_cpu(uint32_t gpu_node); + +HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props); +HSAKMT_STATUS topology_get_node_props(HSAuint32 NodeId, + HsaNodeProperties *NodeProperties); +HSAKMT_STATUS topology_get_iolink_props(HSAuint32 NodeId, + HSAuint32 NumIoLinks, + HsaIoLinkProperties *IoLinkProperties); +void topology_setup_is_dgpu_param(HsaNodeProperties *props); + +HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags); + +#define MIN(a, b) ({ \ + typeof(a) tmp1 = (a), tmp2 = (b); \ + tmp1 < tmp2 ? tmp1 : tmp2; }) + +#define MAX(a, b) ({ \ + typeof(a) tmp1 = (a), tmp2 = (b); \ + tmp1 > tmp2 ? tmp1 : tmp2; }) + +uint32_t get_num_sysfs_nodes(void); + +bool is_forked_child(void); + +/* Calculate VGPR and SGPR register file size per CU */ +uint32_t get_vgpr_size_per_cu(HSA_ENGINE_ID id); +#define SGPR_SIZE_PER_CU 0x4000 + +#endif diff --git a/libhsakmt.ver b/libhsakmt.ver new file mode 100644 index 0000000000..a203bbc90a --- /dev/null +++ b/libhsakmt.ver @@ -0,0 +1,97 @@ +HSAKMT_1 +{ +global: +hsaKmtOpenKFD; +hsaKmtCloseKFD; +hsaKmtGetVersion; +hsaKmtAcquireSystemProperties; +hsaKmtReleaseSystemProperties; +hsaKmtGetNodeProperties; +hsaKmtGetNodeMemoryProperties; +hsaKmtGetNodeCacheProperties; +hsaKmtGetNodeIoLinkProperties; +hsaKmtCreateEvent; +hsaKmtDestroyEvent; +hsaKmtSetEvent; +hsaKmtResetEvent; +hsaKmtQueryEventState; +hsaKmtWaitOnEvent; +hsaKmtWaitOnMultipleEvents; +hsaKmtCreateQueue; +hsaKmtUpdateQueue; +hsaKmtDestroyQueue; +hsaKmtSetQueueCUMask; +hsaKmtSetMemoryPolicy; +hsaKmtAllocMemory; +hsaKmtAllocMemoryAlign; +hsaKmtFreeMemory; +hsaKmtAvailableMemory; +hsaKmtRegisterMemory; +hsaKmtRegisterMemoryToNodes; +hsaKmtRegisterMemoryWithFlags; +hsaKmtRegisterGraphicsHandleToNodes; +hsaKmtShareMemory; +hsaKmtRegisterSharedHandle; +hsaKmtRegisterSharedHandleToNodes; +hsaKmtProcessVMRead; +hsaKmtProcessVMWrite; +hsaKmtDeregisterMemory; +hsaKmtMapMemoryToGPU; +hsaKmtMapMemoryToGPUNodes; +hsaKmtUnmapMemoryToGPU; +hsaKmtDbgRegister; +hsaKmtDbgUnregister; +hsaKmtDbgWavefrontControl; +hsaKmtDbgAddressWatch; +hsaKmtDbgEnable; +hsaKmtDbgDisable; +hsaKmtDbgGetDeviceData; +hsaKmtDbgGetQueueData; +hsaKmtGetClockCounters; +hsaKmtPmcGetCounterProperties; +hsaKmtPmcRegisterTrace; +hsaKmtPmcUnregisterTrace; +hsaKmtPmcAcquireTraceAccess; +hsaKmtPmcReleaseTraceAccess; +hsaKmtPmcStartTrace; +hsaKmtPmcQueryTrace; +hsaKmtPmcStopTrace; +hsaKmtMapGraphicHandle; +hsaKmtUnmapGraphicHandle; +hsaKmtSetTrapHandler; +hsaKmtGetTileConfig; +hsaKmtQueryPointerInfo; +hsaKmtSetMemoryUserData; +hsaKmtGetQueueInfo; +hsaKmtAllocQueueGWS; +hsaKmtRuntimeEnable; +hsaKmtRuntimeDisable; +hsaKmtCheckRuntimeDebugSupport; +hsaKmtGetRuntimeCapabilities; +hsaKmtDebugTrapIoctl; +hsaKmtSPMAcquire; +hsaKmtSPMRelease; +hsaKmtSPMSetDestBuffer; +hsaKmtSVMSetAttr; +hsaKmtSVMGetAttr; +hsaKmtSetXNACKMode; +hsaKmtGetXNACKMode; +hsaKmtOpenSMI; +hsaKmtExportDMABufHandle; +hsaKmtWaitOnEvent_Ext; +hsaKmtWaitOnMultipleEvents_Ext; +hsaKmtReplaceAsanHeaderPage; +hsaKmtReturnAsanHeaderPage; +hsaKmtGetAMDGPUDeviceHandle; +hsaKmtPcSamplingQueryCapabilities; +hsaKmtPcSamplingCreate; +hsaKmtPcSamplingDestroy; +hsaKmtPcSamplingStart; +hsaKmtPcSamplingStop; +hsaKmtPcSamplingSupport; +hsaKmtGetVersionCapInfo; +hsaKmtQueueRingDoorbell; +hsaKmtamdgpu_query_gpu_info; +local: *; +}; + diff --git a/memory.cpp b/memory.cpp new file mode 100644 index 0000000000..b6e35af6fa --- /dev/null +++ b/memory.cpp @@ -0,0 +1,554 @@ +/* + * Copyright © 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "libhsakmt.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "inc/wddm/gpu_memory.h" + +struct Allocation { + Allocation() + : handle(0), cpu_addr(0), gpu_addr(0), size(0), userptr(false), + user_data(nullptr), size_requested(0), node_id(0), mem_flags_value(0) {} + Allocation(rocr::core::GpuMemoryHandle handle_arg, void *cpu_addr_arg, + uint64_t gpu_addr_arg, size_t size_arg, bool userptr_arg = false, + void *user_data_arg = nullptr, size_t user_size_arg = 0, + HSAuint32 node_id_arg = 0, HSAuint32 mem_flags_value_arg = 0) + : handle(handle_arg), cpu_addr(cpu_addr_arg), gpu_addr(gpu_addr_arg), + size(size_arg), userptr(userptr_arg), user_data(user_data_arg), + size_requested(user_size_arg), node_id(node_id_arg), + mem_flags_value(mem_flags_value_arg) {} + + rocr::core::GpuMemoryHandle handle; + void *cpu_addr; + uint64_t gpu_addr; + bool userptr; + size_t size; /* actual size = align_up(size_requested, granularity) */ + void *user_data; + size_t size_requested; /* size requested by user */ + HSAuint32 node_id; + HSAuint32 mem_flags_value; +}; + +static std::map allocation_map_; +static std::mutex allocation_map_lock_; + +HSAKMT_STATUS HSAKMTAPI hsaKmtSetMemoryPolicy(HSAuint32 Node, + HSAuint32 DefaultPolicy, + HSAuint32 AlternatePolicy, + void *MemoryAddressAlternate, + HSAuint64 MemorySizeInBytes) { + CHECK_DXG_OPEN(); + + assert(false); + return HSAKMT_STATUS_SUCCESS; +} + +HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags) { + switch (pageSizeFlags) { + case HSA_PAGE_SIZE_4KB: + return 4 * 1024; + case HSA_PAGE_SIZE_64KB: + return 64 * 1024; + case HSA_PAGE_SIZE_2MB: + return 2 * 1024 * 1024; + case HSA_PAGE_SIZE_1GB: + return 1024 * 1024 * 1024; + default: + assert(false); + return 4 * 1024; + } +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemory(HSAuint32 PreferredNode, + HSAuint64 SizeInBytes, + HsaMemFlags MemFlags, + void **MemoryAddress) { + return hsaKmtAllocMemoryAlign(PreferredNode, SizeInBytes, 0, MemFlags, + MemoryAddress); +} + +#define POWER_OF_2(x) ((x && (!(x & (x - 1)))) ? 1 : 0) + +bool isSystemMemoryAvailable(HSAuint64 SizeInBytes) { + struct sysinfo info; + if (sysinfo(&info) != 0) + return false; + return SizeInBytes <= info.freeram; +} + +bool isLocalMemoryAvailable(rocr::core::WDDMDevice *dev, + HSAuint64 SizeInBytes) { + return SizeInBytes <= dev->VramAvail(); +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemoryAlign(HSAuint32 PreferredNode, + HSAuint64 SizeInBytes, + HSAuint64 Alignment, + HsaMemFlags MemFlags, + void **MemoryAddress) { + CHECK_DXG_OPEN(); + + if (!MemoryAddress) + return HSAKMT_STATUS_INVALID_PARAMETER; + + if (MemFlags.ui32.FixedAddress) { + if (*MemoryAddress == nullptr) + return HSAKMT_STATUS_INVALID_PARAMETER; + } else + *MemoryAddress = nullptr; + + rocr::core::WDDMDevice *dev = get_wddmdev(1); + if (!dev) + return HSAKMT_STATUS_ERROR; + + rocr::core::GpuMemory *gpu_mem = nullptr; + rocr::core::GpuMemoryCreateInfo create_info{}; + create_info.size = SizeInBytes; + + if (!MemFlags.ui32.NonPaged || zfb_support || MemFlags.ui32.GTTAccess) { + /* If allocate VRAM under ZFB mode */ + if (zfb_support && MemFlags.ui32.NonPaged == 1) + MemFlags.ui32.CoarseGrain = 1; + + create_info.domain = rocr_proxy::AllocDomain::kSystem; + if (!isSystemMemoryAvailable(SizeInBytes)) + return HSAKMT_STATUS_NO_MEMORY; + } else { + create_info.domain = rocr_proxy::AllocDomain::kLocal; + if (!isLocalMemoryAvailable(dev, SizeInBytes)) + return HSAKMT_STATUS_NO_MEMORY; + } + + if (!MemFlags.ui32.CoarseGrain) + create_info.mem_flags = rocr_proxy::kFineGrain; + + // create_info.mem_flags |= rocr_proxy::kKernarg; + create_info.flags.physical_only = MemFlags.ui32.NoAddress; + create_info.flags.interprocess = MemFlags.ui32.NoAddress; + create_info.flags.locked = 0; //!!(alloc_flags & AllocatePinned); + + auto code = dev->CreateGpuMemory(create_info, &gpu_mem); + if (code == ErrorCode::Success) { + *MemoryAddress = reinterpret_cast(gpu_mem->GpuAddress()); + std::lock_guard gard(allocation_map_lock_); + allocation_map_[*MemoryAddress] = Allocation( + gpu_mem->GetGpuMemoryHandle(), *MemoryAddress, (uint64_t)*MemoryAddress, + create_info.size, false, nullptr, SizeInBytes, + MemFlags.ui32.GTTAccess ? 0 : PreferredNode, MemFlags.Value); + return HSAKMT_STATUS_SUCCESS; + } + + return HSAKMT_STATUS_ERROR; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtFreeMemory(void *MemoryAddress, + HSAuint64 SizeInBytes) { + CHECK_DXG_OPEN(); + + if (!MemoryAddress) + return HSAKMT_STATUS_INVALID_PARAMETER; + + rocr::core::GpuMemory *gpu_mem = nullptr; + { + std::lock_guard gard(allocation_map_lock_); + auto it = allocation_map_.find(MemoryAddress); + if (it == allocation_map_.end()) { + return HSAKMT_STATUS_ERROR; + } + + gpu_mem = rocr::core::GpuMemory::Convert(it->second.handle); + allocation_map_.erase(it); + } + + delete gpu_mem; + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtAvailableMemory(HSAuint32 Node, + HSAuint64 *AvailableBytes) { + CHECK_DXG_OPEN(); + + if (!AvailableBytes) + return HSAKMT_STATUS_INVALID_PARAMETER; + + rocr::core::WDDMDevice *dev = get_wddmdev(Node); + if (!dev) + return HSAKMT_STATUS_ERROR; + + *AvailableBytes = dev->VramAvail(); + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemory(void *MemoryAddress, + HSAuint64 MemorySizeInBytes) { + CHECK_DXG_OPEN(); + + assert(false); + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemoryToNodes(void *MemoryAddress, + HSAuint64 MemorySizeInBytes, + HSAuint64 NumberOfNodes, + HSAuint32 *NodeArray) { + CHECK_DXG_OPEN(); + + assert(false); + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemoryWithFlags( + void *MemoryAddress, HSAuint64 MemorySizeInBytes, HsaMemFlags MemFlags) { + CHECK_DXG_OPEN(); + + if (!MemoryAddress) + return HSAKMT_STATUS_INVALID_PARAMETER; + + pr_debug("[%s] address %p\n", __func__, MemoryAddress); + + if (MemFlags.ui32.ExtendedCoherent && MemFlags.ui32.CoarseGrain) + return HSAKMT_STATUS_INVALID_PARAMETER; + + // Registered memory should be ordinary paged host memory. + if ((MemFlags.ui32.HostAccess != 1) || (MemFlags.ui32.NonPaged == 1)) + return HSAKMT_STATUS_NOT_SUPPORTED; + + if (!is_dgpu) + /* TODO: support mixed APU and dGPU configurations */ + return HSAKMT_STATUS_NOT_SUPPORTED; + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterGraphicsHandleToNodes( + HSAuint64 GraphicsResourceHandle, + HsaGraphicsResourceInfo *GraphicsResourceInfo, HSAuint64 NumberOfNodes, + HSAuint32 *NodeArray) { + CHECK_DXG_OPEN(); + + assert(false); + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtExportDMABufHandle(void *MemoryAddress, + HSAuint64 MemorySizeInBytes, + int *DMABufFd, + HSAuint64 *Offset) { + CHECK_DXG_OPEN(); + assert(false); + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI +hsaKmtShareMemory(void *MemoryAddress, HSAuint64 SizeInBytes, + HsaSharedMemoryHandle *SharedMemoryHandle) { + CHECK_DXG_OPEN(); + + assert(false); + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI +hsaKmtRegisterSharedHandle(const HsaSharedMemoryHandle *SharedMemoryHandle, + void **MemoryAddress, HSAuint64 *SizeInBytes) { + CHECK_DXG_OPEN(); + + assert(false); + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterSharedHandleToNodes( + const HsaSharedMemoryHandle *SharedMemoryHandle, void **MemoryAddress, + HSAuint64 *SizeInBytes, HSAuint64 NumberOfNodes, HSAuint32 *NodeArray) { + CHECK_DXG_OPEN(); + + assert(false); + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtProcessVMRead(HSAuint32 Pid, + HsaMemoryRange *LocalMemoryArray, + HSAuint64 LocalMemoryArrayCount, + HsaMemoryRange *RemoteMemoryArray, + HSAuint64 RemoteMemoryArrayCount, + HSAuint64 *SizeCopied) { + pr_err("[%s] Deprecated\n", __func__); + + assert(false); + return HSAKMT_STATUS_NOT_IMPLEMENTED; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtProcessVMWrite(HSAuint32 Pid, + HsaMemoryRange *LocalMemoryArray, + HSAuint64 LocalMemoryArrayCount, + HsaMemoryRange *RemoteMemoryArray, + HSAuint64 RemoteMemoryArrayCount, + HSAuint64 *SizeCopied) { + pr_err("[%s] Deprecated\n", __func__); + + assert(false); + return HSAKMT_STATUS_NOT_IMPLEMENTED; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtDeregisterMemory(void *MemoryAddress) { + CHECK_DXG_OPEN(); + + if (!MemoryAddress) + return HSAKMT_STATUS_INVALID_PARAMETER; + + pr_debug("[%s] address %p\n", __func__, MemoryAddress); + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtMapMemoryToGPU(void *MemoryAddress, + HSAuint64 MemorySizeInBytes, + HSAuint64 *AlternateVAGPU) { + CHECK_DXG_OPEN(); + + if (!MemoryAddress || !AlternateVAGPU) { + pr_err("FIXME: mapping NULL pointer\n"); + return HSAKMT_STATUS_ERROR; + } + + uint64_t start = rocr::AlignDown((uint64_t)MemoryAddress, 4096); + uint64_t end = + rocr::AlignUp((uint64_t)MemoryAddress + MemorySizeInBytes, 4096); + + void *aligned_ptr = (void *)start; + size_t aligned_size = end - start; + + { + std::lock_guard gard(allocation_map_lock_); + // GTT mem + auto it_gtt = allocation_map_.find(aligned_ptr); + if (it_gtt != allocation_map_.end()) { + if (!it_gtt->second.userptr) { + if (it_gtt->second.size >= MemorySizeInBytes) { + *AlternateVAGPU = (uint64_t)MemoryAddress; + return HSAKMT_STATUS_SUCCESS; + } else { + return HSAKMT_STATUS_ERROR; + } + } + } + + // userptr mem + auto it = allocation_map_.find(MemoryAddress); + if (it != allocation_map_.end()) { + if (it->second.userptr && it->second.size >= MemorySizeInBytes) { + *AlternateVAGPU = + (uintptr_t)it->second.gpu_addr + + ((uintptr_t)MemoryAddress - (uintptr_t)it->second.cpu_addr); + return HSAKMT_STATUS_SUCCESS; + } + } + } + + rocr::core::WDDMDevice *dev = get_wddmdev(1); + if (!dev) + return HSAKMT_STATUS_ERROR; + + rocr::core::GpuMemory *gpu_mem = nullptr; + rocr::core::GpuMemoryHandle handle = 0; + uint64_t addr; + rocr::core::GpuMemoryCreateInfo create_info{}; + create_info.domain = rocr_proxy::kUserMemory; + create_info.size = aligned_size; + create_info.user_ptr = aligned_ptr; + + auto code = dev->CreateGpuMemory(create_info, &gpu_mem); + if (code == ErrorCode::Success) { + addr = gpu_mem->GpuAddress(); + handle = gpu_mem->GetGpuMemoryHandle(); + } else { + return HSAKMT_STATUS_ERROR; + } + + { + std::lock_guard guard(allocation_map_lock_); + allocation_map_[MemoryAddress] = + Allocation(handle, aligned_ptr, addr, aligned_size, true, MemoryAddress, + MemorySizeInBytes); + allocation_map_[(void *)addr] = + Allocation(handle, aligned_ptr, addr, aligned_size, true, nullptr, + MemorySizeInBytes); + } + + *AlternateVAGPU = addr + ((uintptr_t)MemoryAddress - (uintptr_t)aligned_ptr); + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtMapMemoryToGPUNodes( + void *MemoryAddress, HSAuint64 MemorySizeInBytes, HSAuint64 *AlternateVAGPU, + HsaMemMapFlags MemMapFlags, HSAuint64 NumberOfNodes, HSAuint32 *NodeArray) { + return hsaKmtMapMemoryToGPU(MemoryAddress, MemorySizeInBytes, AlternateVAGPU); +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtUnmapMemoryToGPU(void *MemoryAddress) { + CHECK_DXG_OPEN(); + + if (!MemoryAddress) { + /* Workaround for runtime bug */ + pr_err("FIXME: Unmapping NULL pointer\n"); + return HSAKMT_STATUS_SUCCESS; + } + + pr_debug("[%s] address %p\n", __func__, MemoryAddress); + + rocr::core::GpuMemoryHandle handle = nullptr; + { + std::lock_guard gard(allocation_map_lock_); + auto it = allocation_map_.find(MemoryAddress); + if (it == allocation_map_.end()) { + return HSAKMT_STATUS_ERROR; + } + + if (!it->second.userptr) { + return HSAKMT_STATUS_SUCCESS; + } + + handle = it->second.handle; + + allocation_map_.erase((void *)it->second.gpu_addr); + allocation_map_.erase(it); + } + auto gpu_mem = rocr::core::GpuMemory::Convert(handle); + delete gpu_mem; + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtMapGraphicHandle(HSAuint32 NodeId, + HSAuint64 GraphicDeviceHandle, + HSAuint64 GraphicResourceHandle, + HSAuint64 GraphicResourceOffset, + HSAuint64 GraphicResourceSize, + HSAuint64 *FlatMemoryAddress) { + /* This API was only ever implemented in KFD for Kaveri and + * was never upstreamed. There are no open-source users of + * this interface. It has been superseded by + * RegisterGraphicsHandleToNodes. + */ + return HSAKMT_STATUS_NOT_IMPLEMENTED; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtUnmapGraphicHandle(HSAuint32 NodeId, + HSAuint64 FlatMemoryAddress, + HSAuint64 SizeInBytes) { + CHECK_DXG_OPEN(); + + assert(false); + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtGetTileConfig(HSAuint32 NodeId, + HsaGpuTileConfig *config) { + CHECK_DXG_OPEN(); + + assert(false); + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtQueryPointerInfo(const void *Pointer, + HsaPointerInfo *PointerInfo) { + CHECK_DXG_OPEN(); + + if (!Pointer || !PointerInfo) + return HSAKMT_STATUS_INVALID_PARAMETER; + + pr_debug("[%s] pointer %p\n", __func__, Pointer); + + memset(PointerInfo, 0, sizeof(HsaPointerInfo)); + + Allocation allocation_info; + { + std::lock_guard gard(allocation_map_lock_); + auto it = allocation_map_.find(Pointer); + if (it == allocation_map_.end()) { + PointerInfo->Type = HSA_POINTER_UNKNOWN; + return HSAKMT_STATUS_ERROR; + } + allocation_info = it->second; + } + + if (allocation_info.userptr) { + PointerInfo->Type = HSA_POINTER_REGISTERED_USER; + PointerInfo->SizeInBytes = allocation_info.size; + } else { + PointerInfo->Type = HSA_POINTER_ALLOCATED; + PointerInfo->SizeInBytes = allocation_info.size_requested; + } + + PointerInfo->Node = allocation_info.node_id; + PointerInfo->MemFlags.Value = allocation_info.mem_flags_value; + PointerInfo->CPUAddress = allocation_info.cpu_addr; + PointerInfo->GPUAddress = allocation_info.gpu_addr; + PointerInfo->UserData = allocation_info.user_data; + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtSetMemoryUserData(const void *Pointer, + void *UserData) { + CHECK_DXG_OPEN(); + + assert(false); + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtReplaceAsanHeaderPage(void *addr) { + assert(false); +#ifdef SANITIZER_AMDGPU + pr_debug("[%s] address %p\n", __func__, addr); + CHECK_DXG_OPEN(); + + return HSAKMT_STATUS_SUCCESS; +#else + return HSAKMT_STATUS_NOT_SUPPORTED; +#endif +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtReturnAsanHeaderPage(void *addr) { + assert(false); +#ifdef SANITIZER_AMDGPU + pr_debug("[%s] address %p\n", __func__, addr); + CHECK_DXG_OPEN(); + + return HSAKMT_STATUS_SUCCESS; +#else + return HSAKMT_STATUS_NOT_SUPPORTED; +#endif +} diff --git a/openclose.cpp b/openclose.cpp new file mode 100644 index 0000000000..b2b3c04460 --- /dev/null +++ b/openclose.cpp @@ -0,0 +1,274 @@ +/* + * Copyright © 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "libhsakmt.h" +#include "inc/hsa/hsa.h" +#include "inc/hsa/hsa_ven_amd_loader.h" + +int (*fn_amdgpu_device_get_fd)(HsaAMDGPUDeviceHandle device_handle); + +hsa_signal_value_t (*fn_hsa_signal_load_relaxed)(hsa_signal_t signal); +hsa_signal_value_t (*fn_hsa_signal_wait_relaxed)( + hsa_signal_t signal, hsa_signal_condition_t condition, + hsa_signal_value_t compare_value, uint64_t timeout_hint, + hsa_wait_state_t wait_state_hint); +void (*fn_hsa_signal_store_screlease)(hsa_signal_t hsa_signal, + hsa_signal_value_t value); +hsa_status_t (*fn_hsa_ven_amd_loader_query_host_address)( + const void *device_address, const void **host_address); + +static const char dxg_device_name[] = "/dev/dxg"; +static pid_t parent_pid = -1; +int hsakmt_debug_level; +bool hsakmt_forked; +static int dxg_fd = -1; + +/* is_forked_child detects when the process has forked since the last + * time this function was called. We cannot rely on pthread_atfork + * because the process can fork without calling the fork function in + * libc (using clone or calling the system call directly). + */ +bool is_forked_child(void) { + pid_t cur_pid; + + if (hsakmt_forked) + return true; + + cur_pid = getpid(); + + if (parent_pid == -1) { + parent_pid = cur_pid; + return false; + } + + if (parent_pid != cur_pid) { + hsakmt_forked = true; + return true; + } + + return false; +} + +/* Callbacks from pthread_atfork */ +static void prepare_fork_handler(void) { pthread_mutex_lock(&hsakmt_mutex); } +static void parent_fork_handler(void) { pthread_mutex_unlock(&hsakmt_mutex); } +static void child_fork_handler(void) { + pthread_mutex_init(&hsakmt_mutex, NULL); + hsakmt_forked = true; +} + +/* Call this from the child process after fork. This will clear all + * data that is duplicated from the parent process, that is not valid + * in the child. + * The topology information is duplicated from the parent is valid + * in the child process so it is not cleared + */ +static void clear_after_fork(void) { + // TODO: fmm_clear_all_mem(); + if (dxg_fd) { + close(dxg_fd); + dxg_fd = -1; + } + dxg_open_count = 0; + parent_pid = -1; + hsakmt_forked = false; +} + +static inline void init_page_size(void) { +#ifndef PAGE_SIZE + PAGE_SIZE = sysconf(_SC_PAGESIZE); +#endif + PAGE_SHIFT = ffs(PAGE_SIZE) - 1; +} + +static HSAKMT_STATUS init_vars_from_env(void) { + char *envvar; + int debug_level; + + /* Normally libraries don't print messages. For debugging purpose, we'll + * print messages if an environment variable, HSAKMT_DEBUG_LEVEL, is set. + */ + hsakmt_debug_level = HSAKMT_DEBUG_LEVEL_DEFAULT; + + envvar = getenv("HSAKMT_DEBUG_LEVEL"); + if (envvar) { + debug_level = atoi(envvar); + if (debug_level >= HSAKMT_DEBUG_LEVEL_ERR && + debug_level <= HSAKMT_DEBUG_LEVEL_DEBUG) + hsakmt_debug_level = debug_level; + } + + /* Check whether to support Zero frame buffer */ + envvar = getenv("HSA_ZFB"); + if (envvar) + zfb_support = atoi(envvar); + + /* Check whether to handle vendor specific aql packet */ + envvar = getenv("WSLKMT_VENDOR_PACKET"); + if (envvar) + vendor_packet_support = atoi(envvar); + + return HSAKMT_STATUS_SUCCESS; +} + +#define _HSAKMT_LOOKUP_SYMS(_sym) \ + do { \ + fn_##_sym = \ + reinterpret_cast(dlsym(RTLD_DEFAULT, #_sym)); \ + if (!fn_##_sym) { \ + pr_err("%s not found - %s\n", #_sym, dlerror()); \ + return HSAKMT_STATUS_ERROR; \ + } \ + } while (0) + +static HSAKMT_STATUS init_symbols(void) { + _HSAKMT_LOOKUP_SYMS(hsa_signal_load_relaxed); + _HSAKMT_LOOKUP_SYMS(hsa_signal_wait_relaxed); + _HSAKMT_LOOKUP_SYMS(hsa_signal_store_screlease); + + hsa_status_t (*fn_hsa_system_get_extension_table)( + uint16_t extension, uint16_t version_major, uint16_t version_minor, + void *table); + _HSAKMT_LOOKUP_SYMS(hsa_system_get_extension_table); + hsa_ven_amd_loader_1_03_pfn_t table; + fn_hsa_system_get_extension_table(HSA_EXTENSION_AMD_LOADER, 1, 3, &table); + fn_hsa_ven_amd_loader_query_host_address = + table.hsa_ven_amd_loader_query_host_address; + + return HSAKMT_STATUS_SUCCESS; +} + +static void load_libdrm_amdgpu(void) { + /* load libdrm_amdgpu */ + int fd; + uint32_t major, minor; + amdgpu_device_handle device_handle; + amdgpu_device_initialize(fd, &major, &minor, &device_handle); +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) { + HSAKMT_STATUS result; + int fd = -1; + HsaSystemProperties sys_props; + char *error; + char *useSvmStr; + + pthread_mutex_lock(&hsakmt_mutex); + + /* If the process has forked, the child process must re-initialize + * it's connection to DXG. Any references tracked by dxg_open_count + * belong to the parent + */ + if (is_forked_child()) + clear_after_fork(); + + if (dxg_open_count == 0) { + static bool atfork_installed = false; + + result = init_symbols(); + if (result != HSAKMT_STATUS_SUCCESS) + goto open_failed; + + load_libdrm_amdgpu(); + + result = init_vars_from_env(); + if (result != HSAKMT_STATUS_SUCCESS) + goto open_failed; + + if (dxg_fd < 0) { + fd = open(dxg_device_name, O_RDWR | O_CLOEXEC); + + if (fd == -1) { + result = HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED; + goto open_failed; + } + + dxg_fd = fd; + } + + init_page_size(); + + useSvmStr = getenv("HSA_USE_SVM"); + is_svm_api_supported = !(useSvmStr && !strcmp(useSvmStr, "0")) && false; + + // result = topology_sysfs_get_system_props(&sys_props); + if (result != HSAKMT_STATUS_SUCCESS) + goto topology_sysfs_failed; + + dxg_open_count = 1; + + if (!atfork_installed) { + /* Atfork handlers cannot be uninstalled and + * must be installed only once. Otherwise + * prepare will deadlock when trying to take + * the same lock multiple times. + */ + pthread_atfork(prepare_fork_handler, parent_fork_handler, + child_fork_handler); + atfork_installed = true; + } + } else { + dxg_open_count++; + result = HSAKMT_STATUS_KERNEL_ALREADY_OPENED; + } + + pthread_mutex_unlock(&hsakmt_mutex); + return result; +topology_sysfs_failed: + close(fd); +open_failed: + pthread_mutex_unlock(&hsakmt_mutex); + + return result; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtCloseKFD(void) { + HSAKMT_STATUS result; + + pthread_mutex_lock(&hsakmt_mutex); + + if (dxg_open_count > 0) { + if (--dxg_open_count == 0) { + close(dxg_fd); + } + + result = HSAKMT_STATUS_SUCCESS; + } else + result = HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED; + + pthread_mutex_unlock(&hsakmt_mutex); + + return result; +} diff --git a/pc_sampling.cpp b/pc_sampling.cpp new file mode 100644 index 0000000000..247726239b --- /dev/null +++ b/pc_sampling.cpp @@ -0,0 +1,73 @@ +/* + * Copyright © 2023 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include "libhsakmt.h" + +HSAKMT_STATUS HSAKMTAPI hsaKmtPcSamplingSupport(void) { + CHECK_DXG_OPEN(); + // Used for profiling tools + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS HSAKMTAPI +hsaKmtPcSamplingQueryCapabilities(HSAuint32 NodeId, void *sample_info, + HSAuint32 sample_info_sz, HSAuint32 *size) { + CHECK_DXG_OPEN(); + // Used for profiling tools + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtPcSamplingCreate(HSAuint32 NodeId, + HsaPcSamplingInfo *sample_info, + HsaPcSamplingTraceId *traceId) { + CHECK_DXG_OPEN(); + // Used for profiling tools + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtPcSamplingDestroy(HSAuint32 NodeId, + HsaPcSamplingTraceId traceId) { + CHECK_DXG_OPEN(); + // Used for profiling tools + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtPcSamplingStart(HSAuint32 NodeId, + HsaPcSamplingTraceId traceId) { + CHECK_DXG_OPEN(); + // Used for profiling tools + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtPcSamplingStop(HSAuint32 NodeId, + HsaPcSamplingTraceId traceId) { + CHECK_DXG_OPEN(); + // Used for profiling tools + return HSAKMT_STATUS_NOT_SUPPORTED; +} diff --git a/perfctr.cpp b/perfctr.cpp new file mode 100644 index 0000000000..9f2b755052 --- /dev/null +++ b/perfctr.cpp @@ -0,0 +1,82 @@ +/* + * Copyright © 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#include "libhsakmt.h" + +HSAKMT_STATUS HSAKMTAPI hsaKmtPmcGetCounterProperties( + HSAuint32 NodeId, HsaCounterProperties **CounterProperties) { + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +/* Registers a set of (HW) counters to be used for tracing/profiling */ +HSAKMT_STATUS HSAKMTAPI hsaKmtPmcRegisterTrace(HSAuint32 NodeId, + HSAuint32 NumberOfCounters, + HsaCounter *Counters, + HsaPmcTraceRoot *TraceRoot) { + CHECK_DXG_OPEN(); + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +/* Unregisters a set of (HW) counters used for tracing/profiling */ + +HSAKMT_STATUS HSAKMTAPI hsaKmtPmcUnregisterTrace(HSAuint32 NodeId, + HSATraceId TraceId) { + CHECK_DXG_OPEN(); + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtPmcAcquireTraceAccess(HSAuint32 NodeId, + HSATraceId TraceId) { + CHECK_DXG_OPEN(); + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtPmcReleaseTraceAccess(HSAuint32 NodeId, + HSATraceId TraceId) { + CHECK_DXG_OPEN(); + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +/* Starts tracing operation on a previously established set of performance + * counters */ +HSAKMT_STATUS HSAKMTAPI hsaKmtPmcStartTrace(HSATraceId TraceId, + void *TraceBuffer, + HSAuint64 TraceBufferSizeBytes) { + CHECK_DXG_OPEN(); + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +/*Forces an update of all the counters that a previously started trace operation + * has registered */ +HSAKMT_STATUS HSAKMTAPI hsaKmtPmcQueryTrace(HSATraceId TraceId) { + CHECK_DXG_OPEN(); + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +/* Stops tracing operation on a previously established set of performance + * counters */ +HSAKMT_STATUS HSAKMTAPI hsaKmtPmcStopTrace(HSATraceId TraceId) { + CHECK_DXG_OPEN(); + return HSAKMT_STATUS_NOT_SUPPORTED; +} diff --git a/queues.cpp b/queues.cpp new file mode 100644 index 0000000000..7206e32e99 --- /dev/null +++ b/queues.cpp @@ -0,0 +1,174 @@ +/* + * Copyright © 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#include +#include "libhsakmt.h" +#include "inc/wddm/device.h" +#include "inc/wddm/queue.h" +#include "hsa-runtime/inc/amd_hsa_signal.h" + +uint32_t get_vgpr_size_per_cu(HSA_ENGINE_ID id) { + uint32_t vgpr_size = 0x40000; + + if (id.ui32.Major >= 11) { + vgpr_size = 0x60000; + } + + return vgpr_size; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue( + HSAuint32 NodeId, HSA_QUEUE_TYPE Type, HSAuint32 QueuePercentage, + HSA_QUEUE_PRIORITY Priority, void *QueueAddress, HSAuint64 QueueSizeInBytes, + HsaEvent *Event, HsaQueueResource *QueueResource) { + HSAKMT_STATUS result; + + CHECK_DXG_OPEN(); + assert(Event == nullptr); + + if (Priority < HSA_QUEUE_PRIORITY_MINIMUM || + Priority > HSA_QUEUE_PRIORITY_MAXIMUM) + return HSAKMT_STATUS_INVALID_PARAMETER; + + rocr::core::WDDMDevice *device_ = get_wddmdev(NodeId); + assert(device_); + + switch (Type) { + case HSA_QUEUE_COMPUTE_AQL: { + assert(QueueResource->ErrorReason == nullptr); + uint64_t pkg_num = QueueSizeInBytes / 64; + uint32_t cmdbuf_size = device_->GetCmdbufSize(); + uint32_t queue_engine = device_->GetComputeEngine(); + bool use_hws = device_->IsHwsEnabled(queue_engine); + auto queue_ = new rocr::core::ComputeQueue( + device_, QueueAddress, pkg_num, + reinterpret_cast *>(QueueResource->Queue_write_ptr_aql), + reinterpret_cast *>(QueueResource->Queue_read_ptr_aql), + QueueResource->ErrorReason, cmdbuf_size, queue_engine, use_hws); + + QueueResource->QueueId = reinterpret_cast(queue_); + // for doorbell_signal.hardware_doorbell_ptr + QueueResource->Queue_DoorBell_aql = queue_->GetDoorbellPtr(); + } break; + case HSA_QUEUE_SDMA: + default: + assert(false); + QueueResource->QueueId = 0; + QueueResource->Queue_DoorBell = nullptr; + break; + } + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtUpdateQueue( + HSA_QUEUEID QueueId, HSAuint32 QueuePercentage, HSA_QUEUE_PRIORITY Priority, + void *QueueAddress, HSAuint64 QueueSize, HsaEvent *Event) { + CHECK_DXG_OPEN(); + + if (Priority < HSA_QUEUE_PRIORITY_MINIMUM || + Priority > HSA_QUEUE_PRIORITY_MAXIMUM) + return HSAKMT_STATUS_INVALID_PARAMETER; + + auto queue_ = reinterpret_cast(QueueId); + if (!queue_) + return HSAKMT_STATUS_INVALID_PARAMETER; + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtDestroyQueue(HSA_QUEUEID QueueId) { + CHECK_DXG_OPEN(); + + auto queue_ = reinterpret_cast(QueueId); + + if (!queue_) + return HSAKMT_STATUS_INVALID_PARAMETER; + + delete queue_; + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtSetQueueCUMask(HSA_QUEUEID QueueId, + HSAuint32 CUMaskCount, + HSAuint32 *QueueCUMask) { + CHECK_DXG_OPEN(); + + auto queue_ = reinterpret_cast(QueueId); + if (!queue_) + return HSAKMT_STATUS_INVALID_PARAMETER; + + if (CUMaskCount == 0 || !QueueCUMask || ((CUMaskCount % 32) != 0)) + return HSAKMT_STATUS_INVALID_PARAMETER; + + pr_debug("%s not implemented\n", __func__); + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtGetQueueInfo(HSA_QUEUEID QueueId, + HsaQueueInfo *QueueInfo) { + CHECK_DXG_OPEN(); + + if (QueueInfo == NULL) + return HSAKMT_STATUS_INVALID_PARAMETER; + memset(QueueInfo, 0, sizeof(*QueueInfo)); + + assert(false); + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtSetTrapHandler(HSAuint32 Node, + void *TrapHandlerBaseAddress, + HSAuint64 TrapHandlerSizeInBytes, + void *TrapBufferBaseAddress, + HSAuint64 TrapBufferSizeInBytes) { + CHECK_DXG_OPEN(); + + pr_debug("%s not implemented\n", __func__); + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtAllocQueueGWS(HSA_QUEUEID QueueId, HSAuint32 nGWS, + HSAuint32 *firstGWS) { + CHECK_DXG_OPEN(); + + auto queue_ = reinterpret_cast(QueueId); + if (!queue_) + return HSAKMT_STATUS_INVALID_PARAMETER; + + assert(false); + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtQueueRingDoorbell(HSA_QUEUEID QueueId) { + CHECK_DXG_OPEN(); + + auto queue_ = reinterpret_cast(QueueId); + if (!queue_) + return HSAKMT_STATUS_INVALID_PARAMETER; + + queue_->RingDoorbell(); + return HSAKMT_STATUS_SUCCESS; +} diff --git a/rocr_proxy/librocr_proxy.a b/rocr_proxy/librocr_proxy.a new file mode 100644 index 0000000000..57b428938c Binary files /dev/null and b/rocr_proxy/librocr_proxy.a differ diff --git a/spm.cpp b/spm.cpp new file mode 100644 index 0000000000..d01b3c3e2d --- /dev/null +++ b/spm.cpp @@ -0,0 +1,48 @@ +/* + * Copyright © 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include "libhsakmt.h" + +HSAKMT_STATUS HSAKMTAPI hsaKmtSPMAcquire(HSAuint32 PreferredNode) { + CHECK_DXG_OPEN(); + // Used for profiling tools + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtSPMSetDestBuffer( + HSAuint32 PreferredNode, HSAuint32 SizeInBytes, HSAuint32 *timeout, + HSAuint32 *SizeCopied, void *DestMemoryAddress, bool *isSPMDataLoss) { + CHECK_DXG_OPEN(); + // Used for profiling tools + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtSPMRelease(HSAuint32 PreferredNode) { + CHECK_DXG_OPEN(); + // Used for profiling tools + return HSAKMT_STATUS_NOT_SUPPORTED; +} diff --git a/svm.cpp b/svm.cpp new file mode 100644 index 0000000000..9a45f89de9 --- /dev/null +++ b/svm.cpp @@ -0,0 +1,52 @@ +/* + * Copyright © 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#include "libhsakmt.h" + +/* Helper functions for calling KFD SVM ioctl */ + +HSAKMT_STATUS HSAKMTAPI hsaKmtSVMSetAttr(void *start_addr, HSAuint64 size, + unsigned int nattr, + HSA_SVM_ATTRIBUTE *attrs) { + CHECK_DXG_OPEN(); + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtSVMGetAttr(void *start_addr, HSAuint64 size, + unsigned int nattr, + HSA_SVM_ATTRIBUTE *attrs) { + CHECK_DXG_OPEN(); + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtSetXNACKMode(HSAint32 enable) { + CHECK_DXG_OPEN(); + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtGetXNACKMode(HSAint32 *enable) { + CHECK_DXG_OPEN(); + *enable = false; + return HSAKMT_STATUS_SUCCESS; +} diff --git a/time.cpp b/time.cpp new file mode 100644 index 0000000000..477d04d0ad --- /dev/null +++ b/time.cpp @@ -0,0 +1,52 @@ +/* + * Copyright © 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#include +#include +#include +#include +#include "libhsakmt.h" +#include "inc/wddm/device.h" + +HSAKMT_STATUS HSAKMTAPI hsaKmtGetClockCounters(HSAuint32 NodeId, + HsaClockCounters *Counters) { + HSAKMT_STATUS result = HSAKMT_STATUS_SUCCESS; + + CHECK_DXG_OPEN(); + + std::memset(Counters, 0, sizeof(*Counters)); + + rocr::core::WDDMDevice *device_ = get_wddmdev(NodeId); + assert(device_); + device_->GetClockCounters(&Counters->GPUClockCounter, nullptr); + + struct timespec ts; + if (clock_gettime(CLOCK_REALTIME, &ts) == 0) + Counters->CPUClockCounter = ts.tv_sec * 1e9 + ts.tv_nsec; + if (clock_gettime(CLOCK_BOOTTIME, &ts) == 0) + Counters->SystemClockCounter = ts.tv_sec * 1e9 + ts.tv_nsec; + Counters->SystemClockFrequencyHz = 1000000000; + + return result; +} diff --git a/topology.cpp b/topology.cpp new file mode 100644 index 0000000000..bfa5a22c01 --- /dev/null +++ b/topology.cpp @@ -0,0 +1,1698 @@ +/* + * Copyright © 2014 Advanced Micro Devices, Inc. + * Copyright 2016-2018 Raptor Engineering, LLC. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "libhsakmt.h" +#include "inc/wddm/types.h" +#include "inc/wddm/device.h" + +/* Number of memory banks added by thunk on top of topology + * This only includes static heaps like LDS, scratch and SVM, + * not for MMIO_REMAP heap. MMIO_REMAP memory bank is reported + * dynamically based on whether mmio aperture was mapped + * successfully on this node. + */ +#define NUM_OF_IGPU_HEAPS 3 +#define NUM_OF_DGPU_HEAPS 3 + +typedef struct { + HsaNodeProperties node; + HsaMemoryProperties *mem; /* node->NumBanks elements */ + HsaCacheProperties *cache; + HsaIoLinkProperties *link; +} node_props_t; + +static HsaSystemProperties *g_system; +static node_props_t *g_props; + +static std::vector wdevices_; +static uint32_t wdevice_num_; +static uint32_t num_sysfs_nodes; + +static int processor_vendor = -1; +/* Supported System Vendors */ +enum SUPPORTED_PROCESSOR_VENDORS { + GENUINE_INTEL = 0, + AUTHENTIC_AMD, + IBM_POWER +}; +/* Adding newline to make the search easier */ +static const char *supported_processor_vendor_name[] = { + "GenuineIntel\n", + "AuthenticAMD\n", + "\n" // POWER requires a different search method +}; + +static HSAKMT_STATUS topology_take_snapshot(void); +static void topology_drop_snapshot(void); + +/* information from /proc/cpuinfo */ +struct proc_cpuinfo { + uint32_t proc_num; /* processor */ + uint32_t apicid; /* apicid */ + char model_name[HSA_PUBLIC_NAME_SIZE]; /* model name */ +}; + +/* CPU cache table for all CPUs on the system. Each entry has the relative CPU + * info and caches connected to that CPU. + */ +typedef struct cpu_cacheinfo { + uint32_t len; /* length of the table = number of online procs */ + int32_t proc_num; /* this cpu's processor number */ + uint32_t num_caches; /* number of caches reported by this cpu */ + HsaCacheProperties *cache_prop; /* a list of cache properties */ +} cpu_cacheinfo_t; + +static void free_properties(node_props_t *props, int size) { + if (props) { + int i; + for (i = 0; i < size; i++) { + free(props[i].mem); + free(props[i].cache); + free(props[i].link); + } + + free(props); + } +} + +/* num_subdirs - find the number of sub-directories in the specified path + * @dirpath - directory path to find sub-directories underneath + * @prefix - only count sub-directory names starting with prefix. + * Use blank string, "", to count all. + * Return - number of sub-directories + */ +static int num_subdirs(char *dirpath, char *prefix) { + int count = 0; + DIR *dirp; + struct dirent *dir; + int prefix_len = strlen(prefix); + + dirp = opendir(dirpath); + if (dirp) { + while ((dir = readdir(dirp)) != 0) { + if ((strcmp(dir->d_name, ".") == 0) || (strcmp(dir->d_name, "..") == 0)) + continue; + if (prefix_len && strncmp(dir->d_name, prefix, prefix_len)) + continue; + count++; + } + closedir(dirp); + } + + return count; +} + +/* fscanf_dec - read a file whose content is a decimal number + * @file [IN ] file to read + * @num [OUT] number in the file + */ +static HSAKMT_STATUS fscanf_dec(char *file, uint32_t *num) { + FILE *fd; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + + fd = fopen(file, "r"); + if (!fd) { + pr_err("Failed to open %s\n", file); + return HSAKMT_STATUS_INVALID_PARAMETER; + } + if (fscanf(fd, "%u", num) != 1) { + pr_err("Failed to parse %s as a decimal.\n", file); + ret = HSAKMT_STATUS_ERROR; + } + + fclose(fd); + return ret; +} + +/* fscanf_str - read a file whose content is a string + * @file [IN ] file to read + * @str [OUT] string in the file + */ +static HSAKMT_STATUS fscanf_str(char *file, char *str) { + FILE *fd; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + + fd = fopen(file, "r"); + if (!fd) { + pr_err("Failed to open %s\n", file); + return HSAKMT_STATUS_INVALID_PARAMETER; + } + if (fscanf(fd, "%s", str) != 1) { + pr_err("Failed to parse %s as a string.\n", file); + ret = HSAKMT_STATUS_ERROR; + } + + fclose(fd); + return ret; +} + +/* fscanf_size - read a file whose content represents size as a string + * @file [IN ] file to read + * @bytes [OUT] sizes in bytes + */ +static HSAKMT_STATUS fscanf_size(char *file, uint32_t *bytes) { + FILE *fd; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + char unit; + int n; + + fd = fopen(file, "r"); + if (!fd) { + pr_err("Failed to open %s\n", file); + return HSAKMT_STATUS_INVALID_PARAMETER; + } + + n = fscanf(fd, "%u%c", bytes, &unit); + if (n < 1) { + pr_err("Failed to parse %s\n", file); + ret = HSAKMT_STATUS_ERROR; + } + + if (n == 2) { + switch (unit) { + case 'K': + *bytes <<= 10; + break; + case 'M': + *bytes <<= 20; + break; + case 'G': + *bytes <<= 30; + break; + default: + ret = HSAKMT_STATUS_ERROR; + break; + } + } + + fclose(fd); + return ret; +} + +/* cpumap_to_cpu_ci - translate shared_cpu_map string + cpuinfo->apicid into + * SiblingMap in cache + * @shared_cpu_map [IN ] shared_cpu_map string + * @cpuinfo [IN ] cpuinfo to get apicid + * @this_cache [OUT] CPU cache to fill in SiblingMap + */ +static void cpumap_to_cpu_ci(char *shared_cpu_map, struct proc_cpuinfo *cpuinfo, + HsaCacheProperties *this_cache) { + int num_hexs, bit; + uint32_t proc, apicid, mask; + char *ch_ptr; + + /* shared_cpu_map is shown as ...X3,X2,X1 Each X is a hex without 0x + * and it's up to 8 characters(32 bits). For the first 32 CPUs(actually + * procs), it's presented in X1. The next 32 is in X2, and so on. + */ + num_hexs = (strlen(shared_cpu_map) + 8) / 9; /* 8 characters + "," */ + ch_ptr = strtok(shared_cpu_map, ","); + while (num_hexs-- > 0) { + mask = strtol(ch_ptr, NULL, 16); /* each X */ + for (bit = 0; bit < 32; bit++) { + if (!((1 << bit) & mask)) + continue; + proc = num_hexs * 32 + bit; + apicid = cpuinfo[proc].apicid; + if (apicid >= HSA_CPU_SIBLINGS) { + pr_warn("SiblingMap buffer %d is too small\n", HSA_CPU_SIBLINGS); + continue; + } + this_cache->SiblingMap[apicid] = 1; + } + ch_ptr = strtok(NULL, ","); + } +} + +/* get_cpu_cache_info - get specified CPU's cache information from sysfs + * @prefix [IN] sysfs path for target cpu cache, + * /sys/devices/system/node/nodeX/cpuY/cache + * @cpuinfo [IN] /proc/cpuinfo data to get apicid + * @cpu_ci: CPU specified. This parameter is an input and also an output. + * [IN] cpu_ci->num_caches: number of index dirs + * [OUT] cpu_ci->cache_info: to store cache info collected + * [OUT] cpu_ci->num_caches: reduces when shared with other cpu(s) + * Return: number of cache reported from this cpu + */ +static int get_cpu_cache_info(const char *prefix, struct proc_cpuinfo *cpuinfo, + cpu_cacheinfo_t *cpu_ci) { + int idx, num_idx, n; + HsaCacheProperties *this_cache; + char path[256], str[256]; + bool is_power9 = false; + + if (processor_vendor == IBM_POWER) { + if (strcmp(cpuinfo[0].model_name, "POWER9") == 0) { + is_power9 = true; + } + } + + this_cache = cpu_ci->cache_prop; + num_idx = cpu_ci->num_caches; + for (idx = 0; idx < num_idx; idx++) { + /* If this cache is shared by multiple CPUs, we only need + * to list it in the first CPU. + */ + if (is_power9) { + // POWER9 has SMT4 + if (cpu_ci->proc_num & 0x3) { + /* proc is not 0,4,8,etc. Skip and reduce the cache count. */ + --cpu_ci->num_caches; + continue; + } + } else { + snprintf(path, 256, "%s/index%d/shared_cpu_list", prefix, idx); + /* shared_cpu_list is shown as n1,n2... or n1-n2,n3-n4... + * For both cases, this cache is listed to proc n1 only. + */ + fscanf_dec(path, (uint32_t *)&n); + if (cpu_ci->proc_num != n) { + /* proc is not n1. Skip and reduce the cache count. */ + --cpu_ci->num_caches; + continue; + } + this_cache->ProcessorIdLow = cpuinfo[cpu_ci->proc_num].apicid; + } + + /* CacheLevel */ + snprintf(path, 256, "%s/index%d/level", prefix, idx); + fscanf_dec(path, &this_cache->CacheLevel); + /* CacheType */ + snprintf(path, 256, "%s/index%d/type", prefix, idx); + + memset(str, 0, sizeof(str)); + fscanf_str(path, str); + if (!strcmp(str, "Data")) + this_cache->CacheType.ui32.Data = 1; + if (!strcmp(str, "Instruction")) + this_cache->CacheType.ui32.Instruction = 1; + if (!strcmp(str, "Unified")) { + this_cache->CacheType.ui32.Data = 1; + this_cache->CacheType.ui32.Instruction = 1; + } + this_cache->CacheType.ui32.CPU = 1; + /* CacheSize */ + snprintf(path, 256, "%s/index%d/size", prefix, idx); + fscanf_size(path, &this_cache->CacheSize); + /* CacheLineSize */ + snprintf(path, 256, "%s/index%d/coherency_line_size", prefix, idx); + fscanf_dec(path, &this_cache->CacheLineSize); + /* CacheAssociativity */ + snprintf(path, 256, "%s/index%d/ways_of_associativity", prefix, idx); + fscanf_dec(path, &this_cache->CacheAssociativity); + /* CacheLinesPerTag */ + snprintf(path, 256, "%s/index%d/physical_line_partition", prefix, idx); + fscanf_dec(path, &this_cache->CacheLinesPerTag); + /* CacheSiblings */ + snprintf(path, 256, "%s/index%d/shared_cpu_map", prefix, idx); + fscanf_str(path, str); + cpumap_to_cpu_ci(str, cpuinfo, this_cache); + + ++this_cache; + } + + return cpu_ci->num_caches; +} + +static HSAKMT_STATUS topology_map_node_id(uint32_t node_id, + rocr::core::WDDMDevice *&device) { + uint32_t idx = node_id; + if ((!wdevices_.size()) || (!node_id) || (node_id >= num_sysfs_nodes)) + return HSAKMT_STATUS_NOT_SUPPORTED; + + device = wdevices_[node_id - 1]; + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props) { + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + bool is_node_supported = true; + uint32_t num_supported_nodes = 0; + + assert(props); + std::memset(props, 0, sizeof(*props)); + + D3DKMT_ADAPTERINFO *adapters; + int num_adapters; + if (rocr::core::WDDMGetAdapters(adapters, num_adapters) != STATUS_SUCCESS) { + pr_err("Failed to get adapters\n"); + ret = HSAKMT_STATUS_ERROR; + goto err; + } + + num_sysfs_nodes = num_adapters + 1; + + for (auto device : wdevices_) + delete device; + wdevices_.clear(); + + for (uint32_t i = 0; i < num_adapters; i++) { + rocr::core::WDDMDevice *device = new rocr::core::WDDMDevice( + adapters[i].hAdapter, adapters[i].AdapterLuid); + assert(device && "Create WDDM Device fail"); + wdevices_.push_back(device); + } + props->NumNodes = num_sysfs_nodes; + + delete[] adapters; + return ret; +err: + return ret; +} + +void topology_setup_is_dgpu_param(HsaNodeProperties *props) { + /* if we found a dGPU node, then treat the whole system as dGPU */ + if (!props->NumCPUCores && props->NumFComputeCores) + is_dgpu = true; +} + +static HSAKMT_STATUS topology_get_cpu_model_name(HsaNodeProperties *props, + struct proc_cpuinfo *cpuinfo, + int num_procs) { + int i, j; + + if (!props) { + pr_err("Invalid props to get cpu model name\n"); + return HSAKMT_STATUS_INVALID_PARAMETER; + } + + for (i = 0; i < num_procs; i++, cpuinfo++) { + if (props->CComputeIdLo == cpuinfo->apicid) { + if (!props->DeviceId) /* CPU-only node */ + strncpy((char *)props->AMDName, cpuinfo->model_name, + sizeof(props->AMDName)); + /* Convert from UTF8 to UTF16 */ + for (j = 0; + cpuinfo->model_name[j] != '\0' && j < HSA_PUBLIC_NAME_SIZE - 1; j++) + props->MarketingName[j] = cpuinfo->model_name[j]; + props->MarketingName[j] = '\0'; + return HSAKMT_STATUS_SUCCESS; + } + } + + return HSAKMT_STATUS_ERROR; +} + +static int topology_search_processor_vendor(const char *processor_name) { + unsigned int i; + + for (i = 0; i < ARRAY_LEN(supported_processor_vendor_name); i++) { + if (!strcmp(processor_name, supported_processor_vendor_name[i])) + return i; + if (!strcmp(processor_name, "POWER9, altivec supported\n")) + return IBM_POWER; + } + return -1; +} + +/* topology_parse_cpuinfo - Parse /proc/cpuinfo and fill up required + * topology information + * cpuinfo [OUT]: output buffer to hold cpu information + * num_procs: number of processors the output buffer can hold + */ +static HSAKMT_STATUS topology_parse_cpuinfo(struct proc_cpuinfo *cpuinfo, + uint32_t num_procs) { + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + FILE *fd; + char read_buf[256]; + char *p; + uint32_t proc = 0; + size_t p_len; + const char *proc_cpuinfo_path = "/proc/cpuinfo"; + + if (!cpuinfo) { + pr_err("CPU information will be missing\n"); + return HSAKMT_STATUS_INVALID_PARAMETER; + } + + fd = fopen(proc_cpuinfo_path, "r"); + if (!fd) { + pr_err("Failed to open [%s]. Unable to get CPU information", + proc_cpuinfo_path); + return HSAKMT_STATUS_ERROR; + } + +#ifdef __PPC64__ + char *p2; + + /* Each line in /proc/cpuinfo that read_buf is constructed, the format + * is like this: + * "token : value\n" + * where token is our target like vendor_id, model name, apicid ... + * and value is the answer + */ + while (fgets(read_buf, sizeof(read_buf), fd)) { + /* processor number */ + if (!strncmp("processor ", read_buf, sizeof("processor ") - 1)) { + p = strchr(read_buf, ':'); + p += 2; /* remove ": " */ + proc = atoi(p); + if (proc >= num_procs) { + pr_warn("cpuinfo contains processor %d larger than %u\n", proc, + num_procs); + ret = HSAKMT_STATUS_NO_MEMORY; + goto exit; + } + continue; + } + + /* vendor name / model name */ + if (!strncmp("cpu ", read_buf, sizeof("cpu ") - 1) && + (processor_vendor == -1)) { + p = strchr(read_buf, ':'); + p += 2; /* remove ": " */ + processor_vendor = topology_search_processor_vendor(p); + + p2 = strchr(p, ','); + if (p2 != NULL) { + p2++; + *p2 = 0; + } + if (strlen(p) < HSA_PUBLIC_NAME_SIZE) { + /* -1 to remove \n from p */ + strncpy(cpuinfo[proc].model_name, p, strlen(p) - 1); + cpuinfo[proc].model_name[strlen(p) - 1] = '\0'; + } else + strncpy(cpuinfo[proc].model_name, p, HSA_PUBLIC_NAME_SIZE); + continue; + } + } +#else + /* Each line in /proc/cpuinfo that read_buf is constructed, the format + * is like this: + * "token : value\n" + * where token is our target like vendor_id, model name, apicid ... + * and value is the answer + */ + while (fgets(read_buf, sizeof(read_buf), fd)) { + /* processor number */ + if (!strncmp("processor", read_buf, sizeof("processor") - 1)) { + p = strchr(read_buf, ':'); + p += 2; /* remove ": " */ + proc = atoi(p); + if (proc >= num_procs) { + pr_warn("cpuinfo contains processor %d larger than %u\n", proc, + num_procs); + ret = HSAKMT_STATUS_NO_MEMORY; + goto exit; + } + continue; + } + + /* vendor name */ + if (!strncmp("vendor_id", read_buf, sizeof("vendor_id") - 1) && + (processor_vendor == -1)) { + p = strchr(read_buf, ':'); + p += 2; /* remove ": " */ + processor_vendor = topology_search_processor_vendor(p); + continue; + } + + /* model name */ + if (!strncmp("model name", read_buf, sizeof("model name") - 1)) { + p = strchr(read_buf, ':'); + p += 2; /* remove ": " */ + p_len = strlen(p); + if (p_len > HSA_PUBLIC_NAME_SIZE) + p_len = HSA_PUBLIC_NAME_SIZE; + memcpy(cpuinfo[proc].model_name, p, p_len); + cpuinfo[proc].model_name[p_len - 1] = '\0'; + continue; + } + + /* apicid */ + if (!strncmp("apicid", read_buf, sizeof("apicid") - 1)) { + p = strchr(read_buf, ':'); + p += 2; /* remove ": " */ + cpuinfo[proc].apicid = atoi(p); + } + } +#endif + + if (processor_vendor < 0) { + pr_err("Failed to get Processor Vendor. Setting to %s", + supported_processor_vendor_name[GENUINE_INTEL]); + processor_vendor = GENUINE_INTEL; + } + +exit: + fclose(fd); + return ret; +} + +static HSAKMT_STATUS topology_get_cpu_maxfreq(uint32_t *max_freq) { + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + + std::ifstream cpuinfo_max_freq( + "/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq"); + if (!cpuinfo_max_freq) { + std::ifstream cpuinfo("/proc/cpuinfo"); + if (!cpuinfo) { + std::cerr << "Failed to open /proc/cpuinfo\n"; + return HSAKMT_STATUS_ERROR; + } + + std::string line; + double freq_max_ = 0; + while (std::getline(cpuinfo, line)) { + if (line.substr(0, 7) == "cpu MHz") { + double freq = std::stod(line.substr(line.find(':') + 2)); + if (freq > freq_max_) { + freq_max_ = freq; + } + } + } + *max_freq = static_cast(freq_max_); + } else { + std::string line; + std::getline(cpuinfo_max_freq, line); + *max_freq = static_cast(std::stod(line) / 1000); + } + + return ret; +} + +static int log2_int(int x) { + int result = 0; + while (x >>= 1) { + result++; + } + return result; +} + +static HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, + HsaNodeProperties *props, + bool *p2p_links, + uint32_t *num_p2pLinks) { + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + + assert(props); + memset(props, 0, sizeof(*props)); + if (p2p_links) + *p2p_links = false; + if (num_p2pLinks) + *num_p2pLinks = 0; + + topology_get_cpu_maxfreq(&props->MaxEngineClockMhzCCompute); + + if (node_id == 0) { + /* CPU node */ + props->NumCPUCores = sysconf(_SC_NPROCESSORS_ONLN); + props->NumMemoryBanks = 1; + props->KFDGpuID = 0; + return HSAKMT_STATUS_SUCCESS; + } + + /* gpu node */ + rocr::core::WDDMDevice *device; + ret = topology_map_node_id(node_id, device); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + + props->NumCPUCores = 0; + props->NumFComputeCores = device->SimdPerCu() * device->ComputeUnitCount(); + props->NumMemoryBanks = 1; + props->NumCaches = 3; + props->NumIOLinks = 1; + props->CComputeIdLo = 0; + props->FComputeIdLo = 0; + props->Capability.ui32.ASICRevision = device->AsicRevision(); + props->Capability.ui32.WatchPointsTotalBits = + log2_int(device->WatchPointsNum()); + props->MaxWavesPerSIMD = device->WavePerCu() / device->SimdPerCu(); + props->LDSSizeInKB = device->LdsSize() / 1024; + props->GDSSizeInKB = 0; + props->WaveFrontSize = device->WavefrontSize(); + props->NumShaderBanks = device->NumShaderEngine(); + props->NumArrays = device->ShaderArrayPerShaderEngine(); + props->NumCUPerArray = device->ComputeUnitCount() / props->NumArrays; + props->NumSIMDPerCU = device->SimdPerCu(); + props->MaxSlotsScratchCU = device->MaxScratchSlotsPerCu(); + props->VendorId = 0x1002; + props->DeviceId = device->DeviceId(); + props->LocationId = device->PciBusAddr(); + props->LocalMemSize = 0; + props->MaxEngineClockMhzFCompute = device->MaxEngineClockMhz(); + props->DrmRenderMinor = node_id; + + { + int i; + const char *name = device->ProductName(); + for (i = 0; name[i] != 0 && i < HSA_PUBLIC_NAME_SIZE - 1; i++) + props->MarketingName[i] = name[i]; + props->MarketingName[i] = '\0'; + } + props->uCodeEngineVersions.uCodeSDMA = device->GetSdmaFwVersion(); + props->DebugProperties.Value = 0; + props->HiveID = 0; + props->NumSdmaEngines = device->NumSdmaEngine(); + props->NumSdmaXgmiEngines = 0; + props->NumSdmaQueuesPerEngine = 6; // TODO + props->NumCpQueues = device->GetNumCpQueues(); + props->NumGws = 0; + props->Integrated = !(device->IsDgpu()); + props->Domain = device->Domain(); + props->UniqueID = atol(device->Uuid()); // TODO + props->NumXcc = 1; + props->KFDGpuID = device->DeviceId(); // TODO + props->FamilyID = device->GfxFamily(); + + props->EngineId.ui32.uCode = device->GetMecFwVersion(); + char *envvar = getenv("HSA_OVERRIDE_GFX_VERSION"); + if (envvar) { + char dummy = '\0'; + uint32_t major = 0, minor = 0, step = 0; + /* HSA_OVERRIDE_GFX_VERSION=major.minor.stepping */ + if ((sscanf(envvar, "%u.%u.%u%c", &major, &minor, &step, &dummy) != 3) || + (major > 63 || minor > 255 || step > 255)) { + pr_err("HSA_OVERRIDE_GFX_VERSION %s is invalid\n", envvar); + return HSAKMT_STATUS_ERROR; + } + props->EngineId.ui32.Major = major & 0x3f; + props->EngineId.ui32.Minor = minor & 0xff; + props->EngineId.ui32.Stepping = step & 0xff; + } else { + props->EngineId.ui32.Major = device->Major(); + props->EngineId.ui32.Minor = device->Minor(); + props->EngineId.ui32.Stepping = device->Stepping(); + } + + snprintf((char *)props->AMDName, sizeof(props->AMDName) - 1, "GFX%06x", + HSA_GET_GFX_VERSION_FULL(props->EngineId.ui32)); + + if (!is_svm_api_supported) + props->Capability.ui32.SVMAPISupported = 0; + props->Capability.ui32.DoorbellType = 2; + + /* Get VGPR/SGPR size in byte per CU */ + props->SGPRSizePerCU = SGPR_SIZE_PER_CU; + props->VGPRSizePerCU = get_vgpr_size_per_cu(props->EngineId); + + if (props->NumFComputeCores) + assert(props->EngineId.ui32.Major && + "HSA_OVERRIDE_GFX_VERSION may be needed"); + + return ret; +} + +static HSAKMT_STATUS topology_sysfs_get_mem_props(uint32_t node_id, + uint32_t mem_id, + HsaMemoryProperties *props) { + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + + assert(props); + std::memset(props, 0, sizeof(*props)); + if (node_id == 0) { + /* CPU node */ + props->HeapType = HSA_HEAPTYPE_SYSTEM; + + struct sysinfo info; + sysinfo(&info); + props->SizeInBytes = info.totalram; + + props->Flags.MemoryProperty = 0; + props->Width = 64; + props->MemoryClockMax = 2133; + return HSAKMT_STATUS_SUCCESS; + } + + rocr::core::WDDMDevice *device; + ret = topology_map_node_id(node_id, device); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + + props->HeapType = HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE; + props->SizeInBytes = device->LocalHeapSize(); + props->Width = device->MemoryBusWidth(); + props->MemoryClockMax = device->MaxMemoryClockMhz(); + + return ret; +} + +/* topology_destroy_temp_cpu_cache_list - + * Free the memory allocated in topology_create_temp_cpu_cache_list(). + */ +static void +topology_destroy_temp_cpu_cache_list(cpu_cacheinfo_t *temp_cpu_ci_list) { + uint32_t n; + cpu_cacheinfo_t *p_temp_cpu_ci_list = temp_cpu_ci_list; + cpu_cacheinfo_t *cpu_ci = p_temp_cpu_ci_list; + + if (p_temp_cpu_ci_list) { + for (n = 0; n < p_temp_cpu_ci_list->len; n++, cpu_ci++) + free(cpu_ci->cache_prop); + free(p_temp_cpu_ci_list); + } + + p_temp_cpu_ci_list = NULL; +} + +/* topology_create_temp_cpu_cache_list - Create a temporary cpu-cache list to + * store cpu cache information. This list will be used to copy + * HsaCacheProperties in the CPU node. Two buffers are allocated + * inside this function: cpu_ci list and cache_prop under each + * cpu_ci. Must call topology_destroy_temp_cpu_cache_list to free + * the memory after the information is copied. + * @node [IN] CPU node number + * @cpuinfo [IN] /proc/cpuinfo data + * @temp_cpu_ci_list [OUT] cpu-cache-info list with data filled + * Return: total number of caches under this CPU node + */ +static int +topology_create_temp_cpu_cache_list(int node, struct proc_cpuinfo *cpuinfo, + cpu_cacheinfo_t **temp_cpu_ci_list) { + /* Get max path size from /sys/devices/system/node/node%d/%s/cache + * below, which will max out according to the largest filename, + * which can be present twice in the string above. 29 is for the prefix + * and the +6 is for the cache suffix + */ +#ifndef MAXNAMLEN +/* MAXNAMLEN is the BSD name for NAME_MAX. glibc aliases this as NAME_MAX, but + * not musl */ +#define MAXNAMLEN NAME_MAX +#endif + constexpr uint32_t MAXPATHSIZE = 29 + MAXNAMLEN + (MAXNAMLEN + 6); + cpu_cacheinfo_t *p_temp_cpu_ci_list; /* a list of cpu_ci */ + char path[MAXPATHSIZE], node_dir[MAXPATHSIZE]; + int max_cpus; + cpu_cacheinfo_t *this_cpu; /* one cpu_ci in cpu_ci_list */ + int cache_cnt = 0; + DIR *dirp = NULL; + struct dirent *dir; + char *p; + + if (!temp_cpu_ci_list) { + pr_err("Invalid temp_cpu_ci_list\n"); + return cache_cnt; + } + *temp_cpu_ci_list = NULL; + + /* Get info from /sys/devices/system/node/nodeX/cpuY/cache */ + int node_real = node; + if (processor_vendor == IBM_POWER) { + if (!strcmp(cpuinfo[0].model_name, "POWER9")) { + node_real = node * 8; + } + } + snprintf(node_dir, MAXPATHSIZE, "/sys/devices/system/node/node%d", node_real); + /* Other than cpuY folders, this dir also has cpulist and cpumap */ + max_cpus = num_subdirs(node_dir, "cpu"); + if (max_cpus <= 0) { + /* If CONFIG_NUMA is not enabled in the kernel, + * /sys/devices/system/node doesn't exist. + */ + if (node) { /* CPU node must be 0 or something is wrong */ + pr_err("Fail to get cpu* dirs under %s.", node_dir); + goto exit; + } + /* Fall back to use /sys/devices/system/cpu */ + snprintf(node_dir, MAXPATHSIZE, "/sys/devices/system/cpu"); + max_cpus = num_subdirs(node_dir, "cpu"); + if (max_cpus <= 0) { + pr_err("Fail to get cpu* dirs under %s\n", node_dir); + goto exit; + } + } + + p_temp_cpu_ci_list = + (cpu_cacheinfo_t *)calloc(max_cpus, sizeof(cpu_cacheinfo_t)); + if (!p_temp_cpu_ci_list) { + pr_err("Fail to allocate p_temp_cpu_ci_list\n"); + goto exit; + } + p_temp_cpu_ci_list->len = 0; + + this_cpu = p_temp_cpu_ci_list; + dirp = opendir(node_dir); + while ((dir = readdir(dirp)) != 0) { + if (strncmp(dir->d_name, "cpu", 3)) + continue; + if (!isdigit(dir->d_name[3])) /* ignore files like cpulist */ + continue; + snprintf(path, MAXPATHSIZE, "%s/%s/cache", node_dir, dir->d_name); + this_cpu->num_caches = num_subdirs(path, "index"); + this_cpu->cache_prop = (HsaCacheProperties *)calloc( + this_cpu->num_caches, sizeof(HsaCacheProperties)); + if (!this_cpu->cache_prop) { + pr_err("Fail to allocate cache_info\n"); + goto exit; + } + p = &dir->d_name[3]; + this_cpu->proc_num = atoi(p); + cache_cnt += get_cpu_cache_info(path, cpuinfo, this_cpu); + ++p_temp_cpu_ci_list->len; + ++this_cpu; + } + *temp_cpu_ci_list = p_temp_cpu_ci_list; + +exit: + if (dirp) + closedir(dirp); + return cache_cnt; +} + +/* topology_get_cpu_cache_props - Read CPU cache information from sysfs + * @node [IN] CPU node number + * @cpuinfo [IN] /proc/cpuinfo data + * @tbl [OUT] the node table to fill up + * Return: HSAKMT_STATUS_SUCCESS in success or error number in failure + */ +static HSAKMT_STATUS topology_get_cpu_cache_props(int node, + struct proc_cpuinfo *cpuinfo, + node_props_t *tbl) { + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + cpu_cacheinfo_t *cpu_ci_list = NULL; + uint32_t n, cache_cnt, i; + cpu_cacheinfo_t *cpu_ci; + HsaCacheProperties *this_cache; + + tbl->node.NumCaches = + topology_create_temp_cpu_cache_list(node, cpuinfo, &cpu_ci_list); + if (!tbl->node.NumCaches) { + /* For "Intel Meteor lake Mobile", the cache info is not in sysfs, + * That means /sys/devices/system/node/node%d/%s/cache is not exist. + * here AMD will not black this issue. + */ + pr_debug("CPU cache info is not available for node %d \n", node); + goto exit; + } + + tbl->cache = (HsaCacheProperties *)calloc(tbl->node.NumCaches, + sizeof(HsaCacheProperties)); + if (!tbl->cache) { + ret = HSAKMT_STATUS_NO_MEMORY; + goto exit; + } + + /* Now fill in the information to cache properties. */ + cache_cnt = 0; + cpu_ci = cpu_ci_list; + for (n = 0; n < cpu_ci_list->len; n++, cpu_ci++) { + this_cache = cpu_ci->cache_prop; + for (i = 0; i < cpu_ci->num_caches; i++, this_cache++) { + memcpy(&tbl->cache[cache_cnt++], this_cache, sizeof(HsaCacheProperties)); + if (cache_cnt >= tbl->node.NumCaches) + goto exit; + } + } + +exit: + topology_destroy_temp_cpu_cache_list(cpu_ci_list); + + return ret; +} + +/* For a give Node @node_id the function gets @iolink_id information i.e. parses + * sysfs the following sysfs entry + * ./nodes/@node_id/io_links/@iolink_id/properties. @node_id has to be valid + * accessible node. + * + * If node_to specified by the @iolink_id is not accessible the function returns + * HSAKMT_STATUS_NOT_SUPPORTED. If node_to is accessible, then node_to is mapped + * from sysfs_node to user_node and returns HSAKMT_STATUS_SUCCESS. + */ +static HSAKMT_STATUS topology_sysfs_get_iolink_props(uint32_t node_id, + uint32_t iolink_id, + HsaIoLinkProperties *props, + bool p2pLink) { + rocr::core::WDDMDevice *device; + topology_map_node_id(node_id, device); + + std::memset(props, 0, sizeof(*props)); + props->IoLinkType = HSA_IOLINKTYPE_PCIEXPRESS; + props->VersionMajor = props->VersionMinor = 0; + props->NodeFrom = node_id; + props->NodeTo = 0; + props->Weight = 20; + props->Flags.ui32.Override = 1; + props->Flags.ui32.NonCoherent = 1; + props->Flags.ui32.NoAtomics32bit = !(device->SupportPlatformAtomic()); + props->Flags.ui32.NoAtomics64bit = !(device->SupportPlatformAtomic()); + + return HSAKMT_STATUS_SUCCESS; +} + +/* topology_get_free_io_link_slot_for_node - For the given node_id, find the + * next available free slot to add an io_link + */ +static HsaIoLinkProperties * +topology_get_free_io_link_slot_for_node(uint32_t node_id, + const HsaSystemProperties *sys_props, + node_props_t *node_props) { + HsaIoLinkProperties *props; + + if (node_id >= sys_props->NumNodes) { + pr_err("Invalid node [%d]\n", node_id); + return NULL; + } + + props = node_props[node_id].link; + if (!props) { + pr_err("No io_link reported for Node [%d]\n", node_id); + return NULL; + } + + if (node_props[node_id].node.NumIOLinks >= sys_props->NumNodes - 1) { + pr_err("No more space for io_link for Node [%d]\n", node_id); + return NULL; + } + + return &props[node_props[node_id].node.NumIOLinks]; +} + +/* topology_add_io_link_for_node - If a free slot is available, + * add io_link for the given Node. + * TODO: Add other members of HsaIoLinkProperties + */ +static HSAKMT_STATUS topology_add_io_link_for_node( + uint32_t node_from, const HsaSystemProperties *sys_props, + node_props_t *node_props, HSA_IOLINKTYPE IoLinkType, uint32_t node_to, + uint32_t Weight) { + HsaIoLinkProperties *props; + + props = + topology_get_free_io_link_slot_for_node(node_from, sys_props, node_props); + if (!props) + return HSAKMT_STATUS_NO_MEMORY; + + props->IoLinkType = IoLinkType; + props->NodeFrom = node_from; + props->NodeTo = node_to; + props->Weight = Weight; + node_props[node_from].node.NumIOLinks++; + + return HSAKMT_STATUS_SUCCESS; +} + +/* Find the CPU that this GPU (gpu_node) directly connects to */ +static int32_t gpu_get_direct_link_cpu(uint32_t gpu_node, + node_props_t *node_props) { + HsaIoLinkProperties *props = node_props[gpu_node].link; + uint32_t i; + + if (!node_props[gpu_node].node.KFDGpuID || !props || + node_props[gpu_node].node.NumIOLinks == 0) + return -1; + + for (i = 0; i < node_props[gpu_node].node.NumIOLinks; i++) + if (props[i].IoLinkType == HSA_IOLINKTYPE_PCIEXPRESS && + props[i].Weight <= 20) /* >20 is GPU->CPU->GPU */ + return props[i].NodeTo; + + return -1; +} + +/* Get node1->node2 IO link information. This should be a direct link that has + * been created in the kernel. + */ +static HSAKMT_STATUS get_direct_iolink_info(uint32_t node1, uint32_t node2, + node_props_t *node_props, + HSAuint32 *weight, + HSA_IOLINKTYPE *type) { + HsaIoLinkProperties *props = node_props[node1].link; + uint32_t i; + + if (!props) + return HSAKMT_STATUS_INVALID_NODE_UNIT; + + for (i = 0; i < node_props[node1].node.NumIOLinks; i++) + if (props[i].NodeTo == node2) { + if (weight) + *weight = props[i].Weight; + if (type) + *type = props[i].IoLinkType; + return HSAKMT_STATUS_SUCCESS; + } + + return HSAKMT_STATUS_INVALID_PARAMETER; +} + +static HSAKMT_STATUS get_indirect_iolink_info(uint32_t node1, uint32_t node2, + node_props_t *node_props, + HSAuint32 *weight, + HSA_IOLINKTYPE *type) { + int32_t dir_cpu1 = -1, dir_cpu2 = -1; + HSAuint32 weight1 = 0, weight2 = 0, weight3 = 0; + HSAKMT_STATUS ret; + uint32_t i; + + *weight = 0; + *type = HSA_IOLINKTYPE_UNDEFINED; + + if (node1 == node2) + return HSAKMT_STATUS_INVALID_PARAMETER; + + /* CPU->CPU is not an indirect link */ + if (!node_props[node1].node.KFDGpuID && !node_props[node2].node.KFDGpuID) + return HSAKMT_STATUS_INVALID_NODE_UNIT; + + if (node_props[node1].node.HiveID && node_props[node2].node.HiveID && + node_props[node1].node.HiveID == node_props[node2].node.HiveID) + return HSAKMT_STATUS_INVALID_PARAMETER; + + if (node_props[node1].node.KFDGpuID) + dir_cpu1 = gpu_get_direct_link_cpu(node1, node_props); + if (node_props[node2].node.KFDGpuID) + dir_cpu2 = gpu_get_direct_link_cpu(node2, node_props); + + if (dir_cpu1 < 0 && dir_cpu2 < 0) + return HSAKMT_STATUS_ERROR; + + /* if the node2(dst) is GPU , it need to be large bar for host access*/ + if (node_props[node2].node.KFDGpuID) { + for (i = 0; i < node_props[node2].node.NumMemoryBanks; ++i) + if (node_props[node2].mem[i].HeapType == HSA_HEAPTYPE_FRAME_BUFFER_PUBLIC) + break; + if (i >= node_props[node2].node.NumMemoryBanks) + return HSAKMT_STATUS_ERROR; + } + /* Possible topology: + * GPU --(weight1) -- CPU -- (weight2) -- GPU + * GPU --(weight1) -- CPU -- (weight2) -- CPU -- (weight3) -- GPU + * GPU --(weight1) -- CPU -- (weight2) -- CPU + * CPU -- (weight2) -- CPU -- (weight3) -- GPU + */ + if (dir_cpu1 >= 0) { /* GPU->CPU ... */ + if (dir_cpu2 >= 0) { + if (dir_cpu1 == dir_cpu2) /* GPU->CPU->GPU*/ { + ret = + get_direct_iolink_info(node1, dir_cpu1, node_props, &weight1, NULL); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + ret = + get_direct_iolink_info(dir_cpu1, node2, node_props, &weight2, type); + } else /* GPU->CPU->CPU->GPU*/ { + ret = + get_direct_iolink_info(node1, dir_cpu1, node_props, &weight1, NULL); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + ret = get_direct_iolink_info(dir_cpu1, dir_cpu2, node_props, &weight2, + type); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + /* On QPI interconnection, GPUs can't access + * each other if they are attached to different + * CPU sockets. CPU<->CPU weight larger than 20 + * means the two CPUs are in different sockets. + */ + if (*type == HSA_IOLINK_TYPE_QPI_1_1 && weight2 > 20) + return HSAKMT_STATUS_NOT_SUPPORTED; + ret = + get_direct_iolink_info(dir_cpu2, node2, node_props, &weight3, NULL); + } + } else /* GPU->CPU->CPU */ { + ret = get_direct_iolink_info(node1, dir_cpu1, node_props, &weight1, NULL); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + ret = get_direct_iolink_info(dir_cpu1, node2, node_props, &weight2, type); + } + } else { /* CPU->CPU->GPU */ + ret = get_direct_iolink_info(node1, dir_cpu2, node_props, &weight2, type); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + ret = get_direct_iolink_info(dir_cpu2, node2, node_props, &weight3, NULL); + } + + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + + *weight = weight1 + weight2 + weight3; + return HSAKMT_STATUS_SUCCESS; +} + +static void +topology_create_indirect_gpu_links(const HsaSystemProperties *sys_props, + node_props_t *node_props) { + + uint32_t i, j; + HSAuint32 weight; + HSA_IOLINKTYPE type; + + for (i = 0; i < sys_props->NumNodes - 1; i++) { + for (j = i + 1; j < sys_props->NumNodes; j++) { + get_indirect_iolink_info(i, j, node_props, &weight, &type); + if (!weight) + goto try_alt_dir; + if (topology_add_io_link_for_node(i, sys_props, node_props, type, j, + weight) != HSAKMT_STATUS_SUCCESS) + pr_err("Fail to add IO link %d->%d\n", i, j); + try_alt_dir: + get_indirect_iolink_info(j, i, node_props, &weight, &type); + if (!weight) + continue; + if (topology_add_io_link_for_node(j, sys_props, node_props, type, i, + weight) != HSAKMT_STATUS_SUCCESS) + pr_err("Fail to add IO link %d->%d\n", j, i); + } + } +} + +HSAKMT_STATUS topology_take_snapshot(void) { + uint32_t i, mem_id, cache_id; + HsaSystemProperties sys_props; + node_props_t *temp_props = 0; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + struct proc_cpuinfo *cpuinfo; + const uint32_t num_procs = sysconf(_SC_NPROCESSORS_ONLN); + uint32_t num_ioLinks; + bool p2p_links = false; + uint32_t num_p2pLinks = 0; + + cpuinfo = (proc_cpuinfo *)calloc(num_procs, sizeof(struct proc_cpuinfo)); + if (!cpuinfo) { + pr_err("Fail to allocate memory for CPU info\n"); + return HSAKMT_STATUS_NO_MEMORY; + } + topology_parse_cpuinfo(cpuinfo, num_procs); + + ret = topology_sysfs_get_system_props(&sys_props); + if (ret != HSAKMT_STATUS_SUCCESS) + goto err; + if (sys_props.NumNodes > 0) { + temp_props = + (node_props_t *)calloc(sys_props.NumNodes * sizeof(node_props_t), 1); + if (!temp_props) { + ret = HSAKMT_STATUS_NO_MEMORY; + goto err; + } + for (i = 0; i < sys_props.NumNodes; i++) { + rocr::core::WDDMDevice *device_; + topology_map_node_id(i, device_); + + ret = topology_sysfs_get_node_props(i, &temp_props[i].node, &p2p_links, + &num_p2pLinks); + if (ret != HSAKMT_STATUS_SUCCESS) { + free_properties(temp_props, i); + goto err; + } + + if (temp_props[i].node.NumCPUCores) + topology_get_cpu_model_name(&temp_props[i].node, cpuinfo, num_procs); + + if (temp_props[i].node.NumMemoryBanks) { + temp_props[i].mem = (HsaMemoryProperties *)calloc( + temp_props[i].node.NumMemoryBanks * sizeof(HsaMemoryProperties), 1); + if (!temp_props[i].mem) { + ret = HSAKMT_STATUS_NO_MEMORY; + free_properties(temp_props, i + 1); + goto err; + } + for (mem_id = 0; mem_id < temp_props[i].node.NumMemoryBanks; mem_id++) { + ret = topology_sysfs_get_mem_props(i, mem_id, + &temp_props[i].mem[mem_id]); + if (ret != HSAKMT_STATUS_SUCCESS) { + free_properties(temp_props, i + 1); + goto err; + } + } + } + + if (temp_props[i].node.NumCaches) { + temp_props[i].cache = (HsaCacheProperties *)calloc( + temp_props[i].node.NumCaches * sizeof(HsaCacheProperties), 1); + if (!temp_props[i].cache) { + ret = HSAKMT_STATUS_NO_MEMORY; + free_properties(temp_props, i + 1); + goto err; + } + for (int j = 0; j < 3; j++) { + temp_props[i].cache[j].CacheType.ui32.Data = 1; + temp_props[i].cache[j].CacheType.ui32.HSACU = 1; + temp_props[i].cache[j].CacheLevel = j + 1; + } + temp_props[i].cache[0].CacheSize = device_->GetL1CacheSize() / 1024; + temp_props[i].cache[1].CacheSize = device_->GetL2CacheSize() / 1024; + temp_props[i].cache[2].CacheSize = device_->GetL3CacheSize() / 1024; + } else if (!temp_props[i].node.KFDGpuID) { /* a CPU node */ + ret = topology_get_cpu_cache_props(i, cpuinfo, &temp_props[i]); + if (ret != HSAKMT_STATUS_SUCCESS) { + free_properties(temp_props, i + 1); + goto err; + } + } + + /* To simplify, allocate maximum needed memory for io_links for each node. + * This removes the need for realloc when indirect and QPI links are added + * later + */ + temp_props[i].link = (HsaIoLinkProperties *)calloc( + sys_props.NumNodes - 1, sizeof(HsaIoLinkProperties)); + if (!temp_props[i].link) { + ret = HSAKMT_STATUS_NO_MEMORY; + free_properties(temp_props, i + 1); + goto err; + } + num_ioLinks = temp_props[i].node.NumIOLinks - num_p2pLinks; + uint32_t link_id = 0; + + if (num_ioLinks) { + uint32_t sys_link_id = 0; + + /* Parse all the sysfs specified io links. Skip the ones where the + * remote node (node_to) is not accessible + */ + while (sys_link_id < num_ioLinks && link_id < sys_props.NumNodes - 1) { + ret = topology_sysfs_get_iolink_props( + i, sys_link_id++, &temp_props[i].link[link_id], false); + if (ret == HSAKMT_STATUS_NOT_SUPPORTED) { + ret = HSAKMT_STATUS_SUCCESS; + continue; + } else if (ret != HSAKMT_STATUS_SUCCESS) { + free_properties(temp_props, i + 1); + goto err; + } + link_id++; + } + /* sysfs specifies all the io links. Limit the number to valid ones */ + temp_props[i].node.NumIOLinks = link_id; + } + + if (num_p2pLinks) { + uint32_t sys_link_id = 0; + + /* Parse all the sysfs specified p2p links. + */ + while (sys_link_id < num_p2pLinks && link_id < sys_props.NumNodes - 1) { + ret = topology_sysfs_get_iolink_props( + i, sys_link_id++, &temp_props[i].link[link_id], true); + if (ret == HSAKMT_STATUS_NOT_SUPPORTED) { + ret = HSAKMT_STATUS_SUCCESS; + continue; + } else if (ret != HSAKMT_STATUS_SUCCESS) { + free_properties(temp_props, i + 1); + goto err; + } + link_id++; + } + temp_props[i].node.NumIOLinks = link_id; + } + } + } + + if (!p2p_links) { + /* All direct IO links are created in the kernel. Here we need to + * connect GPU<->GPU or GPU<->CPU indirect IO links. + */ + topology_create_indirect_gpu_links(&sys_props, temp_props); + } + + if (!g_system) { + g_system = (HsaSystemProperties *)malloc(sizeof(HsaSystemProperties)); + if (!g_system) { + free_properties(temp_props, sys_props.NumNodes); + ret = HSAKMT_STATUS_NO_MEMORY; + goto err; + } + } + + *g_system = sys_props; + if (g_props) + free(g_props); + g_props = temp_props; +err: + free(cpuinfo); + return ret; +} + +/* Drop the Snashot of the HSA topology information. Assume lock is held. */ +void topology_drop_snapshot(void) { + if (!!g_system != !!g_props) + pr_warn("Probably inconsistency?\n"); + + if (g_props) { + /* Remove state */ + free_properties(g_props, g_system->NumNodes); + g_props = NULL; + } + + free(g_system); + g_system = NULL; + + for (auto device : wdevices_) + delete device; + wdevices_.clear(); +} + +HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id) { + if (!g_props || !g_system || g_system->NumNodes <= nodeid) + return HSAKMT_STATUS_INVALID_NODE_UNIT; + if (gpu_id) + *gpu_id = g_props[nodeid].node.KFDGpuID; + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS gpuid_to_nodeid(uint32_t gpu_id, uint32_t *node_id) { + uint64_t node_idx; + + for (node_idx = 0; node_idx < g_system->NumNodes; node_idx++) { + if (g_props[node_idx].node.KFDGpuID == gpu_id) { + *node_id = node_idx; + return HSAKMT_STATUS_SUCCESS; + } + } + + return HSAKMT_STATUS_INVALID_NODE_UNIT; +} + +HSAKMT_STATUS HSAKMTAPI +hsaKmtAcquireSystemProperties(HsaSystemProperties *SystemProperties) { + HSAKMT_STATUS err = HSAKMT_STATUS_SUCCESS; + + CHECK_DXG_OPEN(); + + if (!SystemProperties) + return HSAKMT_STATUS_INVALID_PARAMETER; + + pthread_mutex_lock(&hsakmt_mutex); + + /* We already have a valid snapshot. Avoid double initialization that + * would leak memory. + */ + if (g_system) { + *SystemProperties = *g_system; + goto out; + } + + err = topology_take_snapshot(); + if (err != HSAKMT_STATUS_SUCCESS) + goto out; + + assert(g_system); + + // err = fmm_init_process_apertures(g_system->NumNodes); + // TODO: Determine if it is a dGPU + is_dgpu = true; + if (err != HSAKMT_STATUS_SUCCESS) + goto init_process_apertures_failed; + + // err = init_process_doorbells(g_system->NumNodes); + if (err != HSAKMT_STATUS_SUCCESS) + goto init_doorbells_failed; + + *SystemProperties = *g_system; + + goto out; + +init_doorbells_failed: + // fmm_destroy_process_apertures(); +init_process_apertures_failed: + topology_drop_snapshot(); + +out: + pthread_mutex_unlock(&hsakmt_mutex); + return err; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtReleaseSystemProperties(void) { + pthread_mutex_lock(&hsakmt_mutex); + + topology_drop_snapshot(); + + pthread_mutex_unlock(&hsakmt_mutex); + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS topology_get_node_props(HSAuint32 NodeId, + HsaNodeProperties *NodeProperties) { + if (!g_system || !g_props || NodeId >= g_system->NumNodes) + return HSAKMT_STATUS_ERROR; + + *NodeProperties = g_props[NodeId].node; + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI +hsaKmtGetNodeProperties(HSAuint32 NodeId, HsaNodeProperties *NodeProperties) { + HSAKMT_STATUS err; + uint32_t gpu_id; + + if (!NodeProperties) + return HSAKMT_STATUS_INVALID_PARAMETER; + + CHECK_DXG_OPEN(); + pthread_mutex_lock(&hsakmt_mutex); + + err = validate_nodeid(NodeId, &gpu_id); + if (err != HSAKMT_STATUS_SUCCESS) + goto out; + + err = topology_get_node_props(NodeId, NodeProperties); + if (err != HSAKMT_STATUS_SUCCESS) + goto out; + /* For CPU only node don't add any additional GPU memory banks. */ + if (gpu_id) { + uint64_t base, limit; + if (!(NodeProperties->Integrated)) + NodeProperties->NumMemoryBanks += NUM_OF_DGPU_HEAPS; + else + NodeProperties->NumMemoryBanks += NUM_OF_IGPU_HEAPS; + // TODO: for apu + /*if (fmm_get_aperture_base_and_limit(FMM_MMIO, gpu_id, &base, + &limit) == HSAKMT_STATUS_SUCCESS) + NodeProperties->NumMemoryBanks += 1;*/ + } + +out: + pthread_mutex_unlock(&hsakmt_mutex); + return err; +} + +HSAKMT_STATUS HSAKMTAPI +hsaKmtGetNodeMemoryProperties(HSAuint32 NodeId, HSAuint32 NumBanks, + HsaMemoryProperties *MemoryProperties) { + HSAKMT_STATUS err = HSAKMT_STATUS_SUCCESS; + uint32_t i; + + if (!MemoryProperties) + return HSAKMT_STATUS_INVALID_PARAMETER; + + CHECK_DXG_OPEN(); + pthread_mutex_lock(&hsakmt_mutex); + + memset(MemoryProperties, 0, NumBanks * sizeof(HsaMemoryProperties)); + for (i = 0; i < MIN(g_props[NodeId].node.NumMemoryBanks, NumBanks); i++) { + assert(g_props[NodeId].mem); + MemoryProperties[i] = g_props[NodeId].mem[i]; + } + + /* The following memory banks does not apply to CPU only node */ + rocr::core::WDDMDevice *device_ = get_wddmdev(NodeId); + if (device_ == nullptr) + goto out; + + /*Add LDS*/ + if (i < NumBanks) { + MemoryProperties[i].HeapType = HSA_HEAPTYPE_GPU_LDS; + MemoryProperties[i].VirtualBaseAddress = device_->SharedApertureBase(); + MemoryProperties[i].SizeInBytes = g_props[NodeId].node.LDSSizeInKB * 1024; + i++; + } + + /* Add SCRATCH */ + if (i < NumBanks) { + MemoryProperties[i].HeapType = HSA_HEAPTYPE_GPU_SCRATCH; + MemoryProperties[i].VirtualBaseAddress = device_->PrivateApertureBase(); + MemoryProperties[i].SizeInBytes = device_->PrivateApertureSize(); + i++; + } + +out: + pthread_mutex_unlock(&hsakmt_mutex); + return err; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeCacheProperties( + HSAuint32 NodeId, HSAuint32 ProcessorId, HSAuint32 NumCaches, + HsaCacheProperties *CacheProperties) { + HSAKMT_STATUS err; + uint32_t i; + + if (!CacheProperties) + return HSAKMT_STATUS_INVALID_PARAMETER; + + CHECK_DXG_OPEN(); + pthread_mutex_lock(&hsakmt_mutex); + + /* KFD ADD page 18, snapshot protocol violation */ + if (!g_system || NodeId >= g_system->NumNodes) { + err = HSAKMT_STATUS_INVALID_NODE_UNIT; + goto out; + } + + if (NumCaches > g_props[NodeId].node.NumCaches) { + err = HSAKMT_STATUS_INVALID_PARAMETER; + goto out; + } + + for (i = 0; i < MIN(g_props[NodeId].node.NumCaches, NumCaches); i++) { + assert(g_props[NodeId].cache); + CacheProperties[i] = g_props[NodeId].cache[i]; + } + + err = HSAKMT_STATUS_SUCCESS; + +out: + pthread_mutex_unlock(&hsakmt_mutex); + return err; +} + +HSAKMT_STATUS topology_get_iolink_props(HSAuint32 NodeId, HSAuint32 NumIoLinks, + HsaIoLinkProperties *IoLinkProperties) { + if (!g_system || !g_props || NodeId >= g_system->NumNodes) + return HSAKMT_STATUS_ERROR; + + memcpy(IoLinkProperties, g_props[NodeId].link, + NumIoLinks * sizeof(*IoLinkProperties)); + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI +hsaKmtGetNodeIoLinkProperties(HSAuint32 NodeId, HSAuint32 NumIoLinks, + HsaIoLinkProperties *IoLinkProperties) { + HSAKMT_STATUS err; + + if (!IoLinkProperties) + return HSAKMT_STATUS_INVALID_PARAMETER; + + CHECK_DXG_OPEN(); + + pthread_mutex_lock(&hsakmt_mutex); + + /* KFD ADD page 18, snapshot protocol violation */ + if (!g_system || NodeId >= g_system->NumNodes) { + err = HSAKMT_STATUS_INVALID_NODE_UNIT; + goto out; + } + + if (NumIoLinks > g_props[NodeId].node.NumIOLinks) { + err = HSAKMT_STATUS_INVALID_PARAMETER; + goto out; + } + + assert(g_props[NodeId].link); + err = topology_get_iolink_props(NodeId, NumIoLinks, IoLinkProperties); + +out: + pthread_mutex_unlock(&hsakmt_mutex); + return err; +} + +uint16_t get_device_id_by_node_id(HSAuint32 node_id) { + if (!g_props || !g_system || g_system->NumNodes <= node_id) + return 0; + + return g_props[node_id].node.DeviceId; +} + +bool prefer_ats(HSAuint32 node_id) { + return g_props[node_id].node.Capability.ui32.HSAMMUPresent && + g_props[node_id].node.NumCPUCores && + g_props[node_id].node.NumFComputeCores; +} + +uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id) { + unsigned int i; + + if (!g_props || !g_system) + return 0; + + for (i = 0; i < g_system->NumNodes; i++) { + if (g_props[i].node.KFDGpuID == gpu_id) + return g_props[i].node.DeviceId; + } + + return 0; +} + +uint32_t get_direct_link_cpu(uint32_t gpu_node) { + HSAuint64 size = 0; + int32_t cpu_id; + HSAuint32 i; + + cpu_id = gpu_get_direct_link_cpu(gpu_node, g_props); + if (cpu_id == -1) + return INVALID_NODEID; + + assert(g_props[cpu_id].mem); + + for (i = 0; i < g_props[cpu_id].node.NumMemoryBanks; i++) + size += g_props[cpu_id].mem[i].SizeInBytes; + + return size ? (uint32_t)cpu_id : INVALID_NODEID; +} + +HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array, + uint32_t NumberOfNodes, + uint32_t *NodeArray) { + HSAKMT_STATUS ret; + unsigned int i; + + if (NumberOfNodes == 0 || !NodeArray || !gpu_id_array) + return HSAKMT_STATUS_INVALID_PARAMETER; + + /* Translate Node IDs to gpu_ids */ + *gpu_id_array = (uint32_t *)malloc(NumberOfNodes * sizeof(uint32_t)); + if (!(*gpu_id_array)) + return HSAKMT_STATUS_NO_MEMORY; + for (i = 0; i < NumberOfNodes; i++) { + ret = validate_nodeid(NodeArray[i], *gpu_id_array + i); + if (ret != HSAKMT_STATUS_SUCCESS) { + free(*gpu_id_array); + break; + } + } + + return ret; +} + +uint32_t get_num_sysfs_nodes(void) { return num_sysfs_nodes; } + +rocr::core::WDDMDevice *get_wddmdev(uint32_t node_id) { + if ((!wdevices_.size()) || (!node_id) || (node_id >= num_sysfs_nodes)) + return nullptr; + + return wdevices_[node_id - 1]; +} diff --git a/util/atomic_helpers.h b/util/atomic_helpers.h new file mode 100644 index 0000000000..89cef6a638 --- /dev/null +++ b/util/atomic_helpers.h @@ -0,0 +1,519 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +/* + Helpers to use native types with C++11 atomic operations. + Fixes GCC builtin functionality for x86 with respect to WC and non-temporal + stores. +*/ +#ifndef HSA_RUNTIME_CORE_UTIL_ATOMIC_HELPERS_H_ +#define HSA_RUNTIME_CORE_UTIL_ATOMIC_HELPERS_H_ + +#include +#include "utils.h" + +//ALWAYS_CONSERVATIVE will very likely overfence your code. +//For use as a debugging aid only. +#define ALWAYS_CONSERVATIVE 0 + +#if !ALWAYS_CONSERVATIVE +#if defined(__x86_64__) || defined(_M_X64) +#define X64_ORDER_WC 1 +#endif +#if X64_ORDER_WC +#include +#endif +#endif + +namespace rocr { +namespace atomic { + +static constexpr int c11ToBuiltInFlags(std::memory_order order) +{ +#if ALWAYS_CONSERVATIVE + return __ATOMIC_RELAXED; +#elif X64_ORDER_WC + return __ATOMIC_RELAXED; +#else + return (order == std::memory_order_relaxed) ? __ATOMIC_RELAXED : + (order == std::memory_order_acquire) ? __ATOMIC_ACQUIRE : + (order == std::memory_order_release) ? __ATOMIC_RELEASE : + (order == std::memory_order_seq_cst) ? __ATOMIC_SEQ_CST : + (order == std::memory_order_consume) ? __ATOMIC_CONSUME : + (order == std::memory_order_acq_rel) ? __ATOMIC_ACQ_REL : + __ATOMIC_SEQ_CST; +#endif +} + +static __forceinline void PreFence(std::memory_order order) { +#if ALWAYS_CONSERVATIVE + switch (order) { + case std::memory_order_release: + case std::memory_order_seq_cst: + case std::memory_order_acq_rel: + __atomic_thread_fence(__ATOMIC_SEQ_CST); + default:; + } +#elif X64_ORDER_WC + switch (order) { + case std::memory_order_release: + case std::memory_order_seq_cst: + case std::memory_order_acq_rel: + _mm_sfence(); + default:; + } +#endif +} + +static __forceinline void PostFence(std::memory_order order) { +#if ALWAYS_CONSERVATIVE + switch (order) { + case std::memory_order_seq_cst: + case std::memory_order_acq_rel: + case std::memory_order_acquire: + __atomic_thread_fence(__ATOMIC_SEQ_CST); + default:; + } +#elif X64_ORDER_WC + switch (order) { + case std::memory_order_seq_cst: + return _mm_mfence(); + case std::memory_order_acq_rel: + case std::memory_order_acquire: + return _mm_lfence(); + default:; + } +#endif +} + +static __forceinline void Fence(std::memory_order order=std::memory_order_seq_cst) { +#if ALWAYS_CONSERVATIVE + __atomic_thread_fence(__ATOMIC_SEQ_CST); +#elif X64_ORDER_WC + switch (order) { + case std::memory_order_seq_cst: + case std::memory_order_acq_rel: + return _mm_mfence(); + case std::memory_order_acquire: + return _mm_lfence(); + case std::memory_order_release: + return _mm_sfence(); + default:; + } +#else + std::atomic_thread_fence(order); +#endif +} + +template +static __forceinline void BasicCheck(const T* ptr) { + constexpr bool value = __atomic_always_lock_free(sizeof(T), 0); + static_assert(value, "Atomic type may not be compatible with peripheral atomics."); +}; + +template +static __forceinline void BasicCheck(const volatile T* ptr) { + constexpr bool value = __atomic_always_lock_free(sizeof(T), 0); + static_assert(value, "Atomic type may not be compatible with peripheral atomics."); +}; + +/// @brief: Load value of type T atomically with specified memory order. +/// @param: ptr(Input), a pointer to type T. +/// @param: order(Input), memory order with atomic load, relaxed by default. +/// @return: T, loaded value. +template +static __forceinline T + Load(const T* ptr, std::memory_order order = std::memory_order_relaxed) { + BasicCheck(ptr); + T ret; + PreFence(order); + __atomic_load(ptr, &ret, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; +} + +/// @brief: function overloading, for more info, see previous one. +/// @param: ptr(Input), a pointer to volatile type T. +/// @param: order(Input), memory order with atomic load, relaxed by default. +/// @return: T, loaded value. +template +static __forceinline T + Load(const volatile T* ptr, + std::memory_order order = std::memory_order_relaxed) { + BasicCheck(ptr); + T ret; + PreFence(order); + __atomic_load(ptr, &ret, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; +} + +/// @brief: Store value of type T with specified memory order. +/// @param: ptr(Input), a pointer to instance which will be stored. +/// @param: val(Input), value to be stored. +/// @param: order(Input), memory order with atomic store, relaxed by default. +/// @return: void. +template +static __forceinline void Store( + T* ptr, T val, std::memory_order order = std::memory_order_relaxed) { + BasicCheck(ptr); + PreFence(order); + __atomic_store(ptr, &val, c11ToBuiltInFlags(order)); + PostFence(order); +} + +/// @brief: Function overloading, for more info, see previous one. +/// @param: ptr(Input), a pointer to volatile instance which will be stored. +/// @param: val(Input), value to be stored. +/// @param: order(Input), memory order with atomic store, relaxed by default. +/// @return: void. +template +static __forceinline void Store( + volatile T* ptr, T val, + std::memory_order order = std::memory_order_relaxed) { + BasicCheck(ptr); + PreFence(order); + __atomic_store(ptr, &val, c11ToBuiltInFlags(order)); + PostFence(order); +} + +/// @brief: Compare and swap value atomically with specified memory order. +/// @param: ptr(Input), a pointer to variable which is operated on. +/// @param: val(Input), value to be stored if condition is satisfied. +/// @param: expected(Input), value which is expected. +/// @param: order(Input), memory order with atomic operation. +/// @return: T, observed value of type T. +template +static __forceinline T + Cas(T* ptr, T val, T expected, + std::memory_order order = std::memory_order_relaxed) { + BasicCheck(ptr); + PreFence(order); + __atomic_compare_exchange(ptr, &expected, &val, false, c11ToBuiltInFlags(order), __ATOMIC_RELAXED); + PostFence(order); + return expected; +} + +/// @brief: Function overloading, for more info, see previous one. +/// @param: ptr(Input), a pointer to volatile variable which is operated on. +/// @param: val(Input), value to be stored if condition is satisfied. +/// @param: expected(Input), value which is expected. +/// @param: order(Input), memory order which is relaxed by default. +/// @return: T, observed value of type T. +template +static __forceinline T + Cas(volatile T* ptr, T val, T expected, + std::memory_order order = std::memory_order_relaxed) { + BasicCheck(ptr); + PreFence(order); + __atomic_compare_exchange(ptr, &expected, &val, false, c11ToBuiltInFlags(order), __ATOMIC_RELAXED); + PostFence(order); + return expected; +} + +/// @brief: Exchange the value atomically with specified memory order. +/// @param: ptr(Input), a pointer to variable which is operated on. +/// @param: val(Input), value to be stored. +/// @param: order(Input), memory order which is relaxed by default. +/// @return: T, the value prior to the exchange. +template +static __forceinline T + Exchange(T* ptr, T val, + std::memory_order order = std::memory_order_relaxed) { + BasicCheck(ptr); + T ret; + PreFence(order); + __atomic_exchange(ptr, &val, &ret, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; +} + +/// @brief: Function overloading, for more info, see previous one. +/// @param: ptr(Input), a pointer to variable which is operated on. +/// @param: val(Input), value to be stored. +/// @param: order(Input), memory order which is relaxed by default. +/// @return: T, the value prior to the exchange. +template +static __forceinline T + Exchange(volatile T* ptr, T val, + std::memory_order order = std::memory_order_relaxed) { + BasicCheck(ptr); + T ret; + PreFence(order); + __atomic_exchange(ptr, &val, &ret, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; +} + +/// @brief: Add value to variable atomically with specified memory order. +/// @param: ptr(Input), a pointer to variable which is operated on. +/// @param: val(Input), value to be added. +/// @param: order(Input), memory order which is relaxed by default. +/// @return: T, the value of the variable prior to the addition. +template +static __forceinline T + Add(T* ptr, T val, std::memory_order order = std::memory_order_relaxed) { + BasicCheck(ptr); + PreFence(order); + T ret = __atomic_fetch_add(ptr, val, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; +} + +/// @brief: Subtract value from the variable atomically with specified memory +/// order. +/// @param: ptr(Input), a pointer to variable which is operated on. +/// @param: val(Input), value to be subtraced. +/// @param: order(Input), memory order which is relaxed by default. +/// @return: T, value of the variable prior to the subtraction. +template +static __forceinline T + Sub(T* ptr, T val, std::memory_order order = std::memory_order_relaxed) { + BasicCheck(ptr); + PreFence(order); + T ret = __atomic_fetch_sub(ptr, val, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; +} + +/// @brief: Bit And operation on variable atomically with specified memory +/// order. +/// @param: ptr(Input), a pointer to variable which is operated on. +/// @param: val(Input), value which is ANDed with variable. +/// @param: order(Input), memory order which is relaxed by default. +/// @return: T, value of variable prior to the operation. +template +static __forceinline T + And(T* ptr, T val, std::memory_order order = std::memory_order_relaxed) { + BasicCheck(ptr); + PreFence(order); + T ret = __atomic_fetch_and(ptr, val, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; +} + +/// @brief: Bit Or operation on variable atomically with specified memory order. +/// @param: ptr(Input), a pointer to variable which is operated on. +/// @param: val(Input), value which is ORed with variable. +/// @param: order(Input), memory order which is relaxed by default. +/// @return: T, value of variable prior to the operation. +template +static __forceinline T + Or(T* ptr, T val, std::memory_order order = std::memory_order_relaxed) { + BasicCheck(ptr); + PreFence(order); + T ret = __atomic_fetch_or(ptr, val, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; +} + +/// @brief: Bit Xor operation on variable atomically with specified memory +/// order. +/// @param: ptr(Input), a pointer to variable which is operated on. +/// @param: val(Input), value which is XORed with variable. +/// @order: order(Input), memory order which is relaxed by default. +/// @return: T, valud of variable prior to the opertaion. +template +static __forceinline T + Xor(T* ptr, T val, std::memory_order order = std::memory_order_relaxed) { + BasicCheck(ptr); + PreFence(order); + T ret = __atomic_fetch_xor(ptr, val, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; +} + +/// @brief: Increase the value of variable atomically with specified memory +/// order. +/// @param: ptr(Input), a pointer to variable which is operated on. +/// @param: order(Input), memory order which is relaxed by default. +/// @return: T, value of variable prior to the operation. +template +static __forceinline T + Increment(T* ptr, std::memory_order order = std::memory_order_relaxed) { + BasicCheck(ptr); + PreFence(order); + T ret = __atomic_fetch_add(ptr, 1, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; +} + +/// @brief: Decrease the value of the variable atomically with specified memory +/// order. +/// @param: ptr(Input), a pointer to variable which is operated on. +/// @param: order(Input), memory order which is relaxed by default. +/// @return: T, value of variable prior to the operation. +template +static __forceinline T + Decrement(T* ptr, std::memory_order order = std::memory_order_relaxed) { + BasicCheck(ptr); + PreFence(order); + T ret = __atomic_fetch_sub(ptr, 1, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; +} + +/// @brief: Add value to variable atomically with specified memory order. +/// @param: ptr(Input), a pointer to volatile variable which is operated on. +/// @param: val(Input), value to be added. +/// @param: order(Input), memory order which is relaxed by default. +/// @return: T, the value of the variable prior to the addition. +template +static __forceinline T + Add(volatile T* ptr, T val, + std::memory_order order = std::memory_order_relaxed) { + BasicCheck(ptr); + PreFence(order); + T ret = __atomic_fetch_add(ptr, val, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; +} + +/// @brief: Subtract value from the variable atomically with specified memory +/// order. +/// @param: ptr(Input), a pointer to volatile variable which is operated on. +/// @param: val(Input), value to be subtraced. +/// @param: order(Input), memory order which is relaxed by default. +/// @return: T, value of the variable prior to the subtraction. +template +static __forceinline T + Sub(volatile T* ptr, T val, + std::memory_order order = std::memory_order_relaxed) { + BasicCheck(ptr); + PreFence(order); + T ret = __atomic_fetch_sub(ptr, val, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; +} + +/// @brief: Bit And operation on variable atomically with specified memory +/// order. +/// @param: ptr(Input), a pointer to volatile variable which is operated on. +/// @param: val(Input), value which is ANDed with variable. +/// @param: order(Input), memory order which is relaxed by default. +/// @return: T, value of variable prior to the operation. +template +static __forceinline T + And(volatile T* ptr, T val, + std::memory_order order = std::memory_order_relaxed) { + BasicCheck(ptr); + PreFence(order); + T ret = __atomic_fetch_and(ptr, val, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; +} + +/// @brief: Bit Or operation on variable atomically with specified memory order. +/// @param: ptr(Input), a pointer to volatile variable which is operated on. +/// @param: val(Input), value which is ORed with variable. +/// @param: order(Input), memory order which is relaxed by default. +/// @return: T, value of variable prior to the operation. +template +static __forceinline T Or(volatile T* ptr, T val, + std::memory_order order = std::memory_order_relaxed) { + BasicCheck(ptr); + PreFence(order); + T ret = __atomic_fetch_or(ptr, val, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; +} + +/// @brief: Bit Xor operation on variable atomically with specified memory +/// order. +/// @param: ptr(Input), a pointer to volatile variable which is operated on. +/// @param: val(Input), value which is XORed with variable. +/// @order: order(Input), memory order which is relaxed by default. +/// @return: T, valud of variable prior to the opertaion. +template +static __forceinline T + Xor(volatile T* ptr, T val, + std::memory_order order = std::memory_order_relaxed) { + BasicCheck(ptr); + PreFence(order); + T ret = __atomic_fetch_xor(ptr, val, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; +} + +/// @brief: Increase the value of variable atomically with specified memory +/// order. +/// @param: ptr(Input), a pointer to volatile variable which is operated on. +/// @param: order(Input), memory order which is relaxed by default. +/// @return: T, value of variable prior to the operation. +template +static __forceinline T + Increment(volatile T* ptr, + std::memory_order order = std::memory_order_relaxed) { + BasicCheck(ptr); + PreFence(order); + T ret = __atomic_fetch_add(ptr, 1, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; +} + +/// @brief: Decrease the value of the variable atomically with specified memory +/// order. +/// @param: ptr(Input), a pointer to volatile variable which is operated on. +/// @param: order(Input), memory order which is relaxed by default. +/// @return: T, value of variable prior to the operation. +template +static __forceinline T + Decrement(volatile T* ptr, + std::memory_order order = std::memory_order_relaxed) { + BasicCheck(ptr); + PreFence(order); + T ret = __atomic_fetch_sub(ptr, 1, c11ToBuiltInFlags(order)); + PostFence(order); + return ret; +} +} // namespace atomic +} // namespace rocr + +#ifdef X64_ORDER_WC +#undef X64_ORDER_WC +#endif + +#ifdef ALWAYS_CONSERVATIVE +#undef ALWAYS_CONSERVATIVE +#endif + +#endif // HSA_RUNTIME_CORE_UTIL_ATOMIC_HELPERS_H_ diff --git a/util/flag.cpp b/util/flag.cpp new file mode 100644 index 0000000000..c0ac8bd970 --- /dev/null +++ b/util/flag.cpp @@ -0,0 +1,226 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2021-2024, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIESd OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +#include "core/util/flag.h" +#include "core/util/utils.h" +#include "core/util/os.h" + +#include +#include +#include +#include +#include + +namespace rocr { +FILE* log_file = stderr; +uint8_t log_flags[8]; + +void log_printf(const char* file, int line, const char* format, ...) { + va_list ap; + std::stringstream str_thrd_id; + str_thrd_id << std::hex << std::this_thread::get_id(); + va_start(ap, format); + char message[4096]; + vsnprintf(message, sizeof(message), format, ap); + va_end(ap); + fprintf(log_file, ":%-25s:%-4d: %010lld us: [pid:%-5d tid:0x%s] [***rocr***] %s\n", + file, line, os::ReadAccurateClock()/1000ULL, os::GetProcessId(), + str_thrd_id.str().c_str(), message); + fflush(log_file); +} + +// split at separators +static std::vector split(std::string& str, char sep) { + std::vector ret; + while (!str.empty()) { + size_t pos = str.find(sep); + if (pos == std::string::npos) { + ret.push_back(str); + return ret; + } + ret.push_back(str.substr(0, pos)); + str.erase(0, pos + 1); + } + return ret; +}; + +// Parse id,id-id,... strings into id lists +static std::vector get_elements(std::string& str, uint32_t maxElement) { + std::vector ret; + MAKE_NAMED_SCOPE_GUARD(error, [&]() { ret.clear(); }); + + std::vector ranges = split(str, ','); + for (auto& str : ranges) { + auto range = split(str, '-'); + // failure, too many -'s. + if (range.size() > 2) return ret; + + char* end; + uint32_t index = strtoul(range[0].c_str(), &end, 10); + // Invalid syntax - id's must be base 10 digits only. + if (*end != '\0') return ret; + if (index <= maxElement) ret.push_back(index); + + if (range.size() == 2) { + uint32_t secondindex = strtoul(range[1].c_str(), &end, 10); + if (*end != '\0') return ret; // bad syntax + if (secondindex < index) return ret; // inverted range + secondindex = Min(secondindex, maxElement); + for (uint32_t i = index + 1; i < secondindex + 1; i++) ret.push_back(i); + } + } + + // Confirm no duplicate ids. + std::sort(ret.begin(), ret.end()); + if (std::adjacent_find(ret.begin(), ret.end()) != ret.end()) return ret; + + // Good parse, keep result. + error.Dismiss(); + return ret; +}; + +/* +Parse env var per the following syntax, all whitespace is ignored: + +ID = [0-9][0-9]* ex. base 10 numbers +ID_list = (ID | ID-ID)[, (ID | ID-ID)]* ex. 0,2-4,7 +GPU_list = ID_list ex. 0,2-4,7 +CU_list = 0x[0-F]* | ID_list ex. 0x337F OR 0,2-4,7 +CU_Set = GPU_list : CU_list ex. 0,2-4,7:0-15,32-47 OR 0,2-4,7:0x337F +HSA_CU_MASK = CU_Set [; CU_Set]* ex. 0,2-4,7:0-15,32-47; 3-9:0x337F + +GPU indexes are taken post ROCR_VISIBLE_DEVICES reordering. +Listed or bit set CUs will be enabled at queue creation on the associated GPU. +All other CUs on the associated GPUs will be disabled. +CU masks of unlisted GPUs are not restricted. + +Repeating a GPU or CU ID is a syntax error. +Parsing stops at the first CU_Set that has a syntax error, that set and all +following sets are ignored. +Specifying a mask with no usable CUs (CU_list is 0x0) is a syntax error. +Users should use ROCR_VISIBLE_DEVICES if they want to exclude use of a +particular GPU. +*/ +void Flag::parse_masks(std::string& var, uint32_t maxGpu, uint32_t maxCU) { + if (var.empty()) return; + + // Remove whitespace + auto end = std::remove_if(var.begin(), var.end(), + [](char c) { return std::isspace(c, std::locale::classic()); }); + var.erase(end, var.end()); + + // Switch to uppercase + for (auto& c : var) c = toupper(c); + + // Iterate over cu sets + auto sets = split(var, ';'); + for (auto& set : sets) { + auto parts = split(set, ':'); + if (parts.size() != 2) return; + + // temp storage for cu_set parsing. + std::vector gpu_index; + std::vector mask; + + // parse cu list first, check for bitmask format + if (parts[1][1] == 'X') { + // Confirm hex format and strip prefix + auto& cu = parts[1]; + if (cu[0] != '0') return; + cu.erase(0, 2); + + // Ensure all valid hex characters + for (auto& c : cu) { + if (!isxdigit(c)) return; + } + + // Convert to uint32_t, lsb first. + size_t len = cu.length(); + while (len != 0) { + size_t trim = Min(len, size_t(8)); + len -= trim; + auto tmp = cu.substr(len, trim); + auto chunk = stoul(tmp, nullptr, 16); + mask.push_back(chunk); + } + + // Trim dwords beyond maxCUs + uint32_t maxDwords = maxCU / 32 + 1; + if (maxDwords < mask.size()) mask.resize(maxDwords); + + // Trim leading zeros + while (!mask.empty() && mask.back() == 0) mask.pop_back(); + + // Mask 0x0 is an error. + if (mask.empty()) return; + + } else { + // parse cu lists + auto cu_indices = get_elements(parts[1], maxCU); + if (cu_indices.empty()) return; + uint32_t maxdword = cu_indices.back() / 32 + 1; + mask.resize(maxdword, 0); + for (auto id : cu_indices) { + uint32_t index, offset; + index = id / 32; + offset = id % 32; + mask[index] |= 1ul << offset; + } + } + + // parse device list + gpu_index = get_elements(parts[0], maxGpu); + if (gpu_index.empty()) return; + + // Ensure that no GPU was repeated across cu_sets + for (auto id : gpu_index) { + if (cu_mask_.find(id) != cu_mask_.end()) return; + } + + // Insert into map + for (auto id : gpu_index) { + cu_mask_[id] = mask; + } + } +} + +} // namespace rocr diff --git a/util/flag.h b/util/flag.h new file mode 100644 index 0000000000..46dcf89cae --- /dev/null +++ b/util/flag.h @@ -0,0 +1,360 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2014-2021, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIESd OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +#ifndef HSA_RUNTIME_CORE_INC_FLAG_H_ +#define HSA_RUNTIME_CORE_INC_FLAG_H_ + +#include + +#include +#include +#include + +#include "core/util/os.h" +#include "core/util/utils.h" + +namespace rocr { + +class Flag { + public: + enum SDMA_OVERRIDE { SDMA_DISABLE, SDMA_ENABLE, SDMA_DEFAULT }; + enum SRAMECC_ENABLE { SRAMECC_DISABLED, SRAMECC_ENABLED, SRAMECC_DEFAULT }; + + // The values are meaningful and chosen to satisfy the thunk API. + enum XNACK_REQUEST { XNACK_DISABLE = 0, XNACK_ENABLE = 1, XNACK_UNCHANGED = 2 }; + static_assert(XNACK_DISABLE == 0, "XNACK_REQUEST enum values improperly changed."); + static_assert(XNACK_ENABLE == 1, "XNACK_REQUEST enum values improperly changed."); + + // Lift limit for 2.10 release RCCL workaround. + const size_t DEFAULT_SCRATCH_SINGLE_LIMIT = 146800640; // small_limit >> 2; + + explicit Flag() { Refresh(); } + + virtual ~Flag() {} + + void Refresh() { + std::string var = os::GetEnvVar("HSA_CHECK_FLAT_SCRATCH"); + check_flat_scratch_ = (var == "1") ? true : false; + + var = os::GetEnvVar("HSA_ENABLE_VM_FAULT_MESSAGE"); + enable_vm_fault_message_ = (var == "0") ? false : true; + + var = os::GetEnvVar("HSA_ENABLE_QUEUE_FAULT_MESSAGE"); + enable_queue_fault_message_ = (var == "0") ? false : true; + + var = os::GetEnvVar("HSA_ENABLE_INTERRUPT"); + enable_interrupt_ = (var == "0") ? false : true; + + var = os::GetEnvVar("HSA_ENABLE_SDMA"); + enable_sdma_ = (var == "0") ? SDMA_DISABLE : ((var == "1") ? SDMA_ENABLE : SDMA_DEFAULT); + + var = os::GetEnvVar("HSA_ENABLE_PEER_SDMA"); + enable_peer_sdma_ = (var == "0") ? SDMA_DISABLE : ((var == "1") ? SDMA_ENABLE : SDMA_DEFAULT); + + var = os::GetEnvVar("HSA_ENABLE_SDMA_GANG"); + enable_sdma_gang_ = (var == "0") ? SDMA_DISABLE : + ((var == "1") ? SDMA_ENABLE : SDMA_DEFAULT); + + var = os::GetEnvVar("HSA_ENABLE_SDMA_COPY_SIZE_OVERRIDE"); + enable_sdma_copy_size_override_ = (var == "0") ? SDMA_DISABLE : + ((var == "1") ? SDMA_ENABLE : SDMA_DEFAULT); + + visible_gpus_ = os::GetEnvVar("ROCR_VISIBLE_DEVICES"); + filter_visible_gpus_ = os::IsEnvVarSet("ROCR_VISIBLE_DEVICES"); + + var = os::GetEnvVar("HSA_RUNNING_UNDER_VALGRIND"); + running_valgrind_ = (var == "1") ? true : false; + + var = os::GetEnvVar("HSA_SDMA_WAIT_IDLE"); + sdma_wait_idle_ = (var == "1") ? true : false; + + var = os::GetEnvVar("HSA_MAX_QUEUES"); + max_queues_ = static_cast(atoi(var.c_str())); + + // Maximum amount of scratch mem that can be used per process per gpu + var = os::GetEnvVar("HSA_SCRATCH_MEM"); + scratch_mem_size_ = atoi(var.c_str()); + + // Scratch memory sizes > HSA_SCRATCH_SINGLE_LIMIT will trigger a use-once scheme + // We also reserve HSA_SCRATCH_SINGLE_LIMIT per process per gpu to guarrantee we + // have sufficient memory to for scratch in case user tried to allocate all device + // memory + if (os::IsEnvVarSet("HSA_SCRATCH_SINGLE_LIMIT")) { + var = os::GetEnvVar("HSA_SCRATCH_SINGLE_LIMIT"); + scratch_single_limit_ = atoi(var.c_str()); + } else { + scratch_single_limit_ = DEFAULT_SCRATCH_SINGLE_LIMIT; + } + + tools_lib_names_ = os::GetEnvVar("HSA_TOOLS_LIB"); + + var = os::GetEnvVar("HSA_TOOLS_REPORT_LOAD_FAILURE"); + + ifdebug { + report_tool_load_failures_ = (var == "1") ? true : false; + } else { + report_tool_load_failures_ = (var == "0") ? false : true; + } + + var = os::GetEnvVar("HSA_DISABLE_FRAGMENT_ALLOCATOR"); + disable_fragment_alloc_ = (var == "1") ? true : false; + + var = os::GetEnvVar("HSA_ENABLE_SDMA_HDP_FLUSH"); + enable_sdma_hdp_flush_ = (var == "0") ? false : true; + + var = os::GetEnvVar("HSA_REV_COPY_DIR"); + rev_copy_dir_ = (var == "1") ? true : false; + + var = os::GetEnvVar("HSA_FORCE_FINE_GRAIN_PCIE"); + fine_grain_pcie_ = (var == "1") ? true : false; + + var = os::GetEnvVar("HSA_NO_SCRATCH_RECLAIM"); + no_scratch_reclaim_ = (var == "1") ? true : false; + + var = os::GetEnvVar("HSA_NO_SCRATCH_THREAD_LIMITER"); + no_scratch_thread_limit_ = (var == "1") ? true : false; + + var = os::GetEnvVar("HSA_DISABLE_IMAGE"); + disable_image_ = (var == "1") ? true : false; + + var = os::GetEnvVar("HSA_DISABLE_PC_SAMPLING"); + disable_pc_sampling_ = (var == "1") ? true : false; + + var = os::GetEnvVar("HSA_LOADER_ENABLE_MMAP_URI"); + loader_enable_mmap_uri_ = (var == "1") ? true : false; + + var = os::GetEnvVar("HSA_FORCE_SDMA_SIZE"); + force_sdma_size_ = var.empty() ? 1024 * 1024 : atoi(var.c_str()); + + var = os::GetEnvVar("HSA_IGNORE_SRAMECC_MISREPORT"); + check_sramecc_validity_ = (var == "1") ? false : true; + + // Legal values are zero "0" or one "1". Any other value will + // be interpreted as not defining the env variable. + var = os::GetEnvVar("HSA_XNACK"); + xnack_ = (var == "0") ? XNACK_DISABLE : ((var == "1") ? XNACK_ENABLE : XNACK_UNCHANGED); + + var = os::GetEnvVar("HSA_ENABLE_DEBUG"); + debug_ = (var == "1") ? true : false; + + var = os::GetEnvVar("HSA_CU_MASK_SKIP_INIT"); + cu_mask_skip_init_ = (var == "1") ? true : false; + + // Temporary opt-in for corrected HSA_AMD_AGENT_INFO_COOPERATIVE_COMPUTE_UNIT_COUNT behavior. + // Will become opt-out and possibly removed in future releases. + var = os::GetEnvVar("HSA_COOP_CU_COUNT"); + coop_cu_count_ = (var == "1") ? true : false; + + var = os::GetEnvVar("HSA_DISCOVER_COPY_AGENTS"); + discover_copy_agents_ = (var == "1") ? true : false; + + var = os::GetEnvVar("HSA_SVM_PROFILE"); + svm_profile_ = var; + + var = os::GetEnvVar("HSA_ENABLE_SRAMECC"); + sramecc_enable_ = + (var == "0") ? SRAMECC_DISABLED : ((var == "1") ? SRAMECC_ENABLED : SRAMECC_DEFAULT); + + var = os::GetEnvVar("HSA_IMAGE_PRINT_SRD"); + image_print_srd_ = (var == "1") ? true : false; + + var = os::GetEnvVar("HSA_ENABLE_MWAITX"); + enable_mwaitx_ = (var == "1") ? true : false; + + // Temporary environment variable to disable CPU affinity override + // Will either rename to HSA_OVERRIDE_CPU_AFFINITY later or remove completely. + var = os::GetEnvVar("HSA_OVERRIDE_CPU_AFFINITY_DEBUG"); + override_cpu_affinity_ = (var == "0") ? false : true; + } + + void parse_masks(uint32_t maxGpu, uint32_t maxCU) { + std::string var = os::GetEnvVar("HSA_CU_MASK"); + parse_masks(var, maxGpu, maxCU); + } + + bool check_flat_scratch() const { return check_flat_scratch_; } + + bool enable_vm_fault_message() const { return enable_vm_fault_message_; } + + bool enable_queue_fault_message() const { return enable_queue_fault_message_; } + + bool enable_interrupt() const { return enable_interrupt_; } + + bool enable_sdma_hdp_flush() const { return enable_sdma_hdp_flush_; } + + bool running_valgrind() const { return running_valgrind_; } + + bool sdma_wait_idle() const { return sdma_wait_idle_; } + + bool report_tool_load_failures() const { return report_tool_load_failures_; } + + bool disable_fragment_alloc() const { return disable_fragment_alloc_; } + + bool rev_copy_dir() const { return rev_copy_dir_; } + + bool fine_grain_pcie() const { return fine_grain_pcie_; } + + bool no_scratch_reclaim() const { return no_scratch_reclaim_; } + + bool no_scratch_thread_limiter() const { return no_scratch_thread_limit_; } + + SDMA_OVERRIDE enable_sdma() const { return enable_sdma_; } + + SDMA_OVERRIDE enable_peer_sdma() const { return enable_peer_sdma_; } + + SDMA_OVERRIDE enable_sdma_gang() const { return enable_sdma_gang_; } + + SDMA_OVERRIDE enable_sdma_copy_size_override() const { return enable_sdma_copy_size_override_; } + + std::string visible_gpus() const { return visible_gpus_; } + + bool filter_visible_gpus() const { return filter_visible_gpus_; } + + uint32_t max_queues() const { return max_queues_; } + + size_t scratch_mem_size() const { return scratch_mem_size_; } + + size_t scratch_single_limit() const { return scratch_single_limit_; } + + std::string tools_lib_names() const { return tools_lib_names_; } + + bool disable_image() const { return disable_image_; } + + bool disable_pc_sampling() const { return disable_pc_sampling_; } + + bool loader_enable_mmap_uri() const { return loader_enable_mmap_uri_; } + + size_t force_sdma_size() const { return force_sdma_size_; } + + bool check_sramecc_validity() const { return check_sramecc_validity_; } + + bool override_cpu_affinity() const { return override_cpu_affinity_; } + + bool image_print_srd() const { return image_print_srd_; } + + bool check_mwaitx(bool mwaitx_supported) { + if (enable_mwaitx_ && !mwaitx_supported) enable_mwaitx_ = false; + + return enable_mwaitx_; + } + + XNACK_REQUEST xnack() const { return xnack_; } + + bool debug() const { return debug_; } + + const std::vector& cu_mask(uint32_t gpu_index) const { + static const std::vector empty; + auto it = cu_mask_.find(gpu_index); + if (it == cu_mask_.end()) return empty; + return it->second; + } + + bool cu_mask_skip_init() const { return cu_mask_skip_init_; } + + bool coop_cu_count() const { return coop_cu_count_; } + + bool discover_copy_agents() const { return discover_copy_agents_; } + + const std::string& svm_profile() const { return svm_profile_; } + + SRAMECC_ENABLE sramecc_enable() const { return sramecc_enable_; } + + private: + bool check_flat_scratch_; + bool enable_vm_fault_message_; + bool enable_interrupt_; + bool enable_sdma_hdp_flush_; + bool running_valgrind_; + bool sdma_wait_idle_; + bool enable_queue_fault_message_; + bool report_tool_load_failures_; + bool disable_fragment_alloc_; + bool rev_copy_dir_; + bool fine_grain_pcie_; + bool no_scratch_reclaim_; + bool no_scratch_thread_limit_; + bool disable_image_; + bool disable_pc_sampling_; + bool loader_enable_mmap_uri_; + bool check_sramecc_validity_; + bool debug_; + bool cu_mask_skip_init_; + bool coop_cu_count_; + bool discover_copy_agents_; + bool override_cpu_affinity_; + bool image_print_srd_; + bool enable_mwaitx_; + + SDMA_OVERRIDE enable_sdma_; + SDMA_OVERRIDE enable_peer_sdma_; + SDMA_OVERRIDE enable_sdma_gang_; + SDMA_OVERRIDE enable_sdma_copy_size_override_; + + bool filter_visible_gpus_; + std::string visible_gpus_; + + uint32_t max_queues_; + + size_t scratch_mem_size_; + size_t scratch_single_limit_; + + std::string tools_lib_names_; + std::string svm_profile_; + + size_t force_sdma_size_; + + // Indicates user preference for Xnack state. + XNACK_REQUEST xnack_; + + SRAMECC_ENABLE sramecc_enable_; + + // Map GPU index post RVD to its default cu mask. + std::map> cu_mask_; + + void parse_masks(std::string& args, uint32_t maxGpu, uint32_t maxCU); + + DISALLOW_COPY_AND_ASSIGN(Flag); +}; + +} // namespace rocr + +#endif // header guard diff --git a/util/lazy_ptr.h b/util/lazy_ptr.h new file mode 100644 index 0000000000..2aef6a3bf3 --- /dev/null +++ b/util/lazy_ptr.h @@ -0,0 +1,155 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIESd OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +#ifndef HSA_RUNTIME_CORE_UTIL_LAZY_PTR_H_ +#define HSA_RUNTIME_CORE_UTIL_LAZY_PTR_H_ + +#include +#include +#include + +#include "core/util/locks.h" +#include "core/util/utils.h" + +namespace rocr { + +/* + * Wrapper for a std::unique_ptr that initializes its object at first use. + */ +template class lazy_ptr { + public: + lazy_ptr() {} + + explicit lazy_ptr(std::function Constructor) { reset(Constructor); } + + lazy_ptr(lazy_ptr&& rhs) { + obj = std::move(rhs.obj); + func = std::move(rhs.func); + } + + lazy_ptr& operator=(lazy_ptr&& rhs) { + obj = std::move(rhs.obj); + func = std::move(rhs.func); + } + + lazy_ptr(lazy_ptr&) = delete; + lazy_ptr& operator=(lazy_ptr&) = delete; + + void reset(std::function Constructor = nullptr) { + obj.reset(); + func = Constructor; + } + + void reset(T* ptr) { + obj.reset(ptr); + func = nullptr; + } + + bool operator==(T* rhs) const { return obj.get() == rhs; } + bool operator!=(T* rhs) const { return obj.get() != rhs; } + + const std::unique_ptr& operator->() const { + make(true); + assert(obj != nullptr && "Null dereference through lazy_ptr."); + return obj; + } + + std::unique_ptr& operator*() { + make(true); + return obj; + } + + const std::unique_ptr& operator*() const { + make(true); + return obj; + } + + /* + * Ensures that the object is created or is being created. + * This is useful when early construction of the object is required. + */ + void touch() const { make(false); } + + // Tells if the lazy object has been constructed or not. + // Construction may fail silently (return nullptr). + bool created() const { + std::atomic_thread_fence(std::memory_order_acquire); + return func == nullptr; + } + + // Tells if the lazy object exists or not. + bool empty() const { + std::atomic_thread_fence(std::memory_order_acquire); + return obj == nullptr; + } + + private: + mutable std::unique_ptr obj; + mutable std::function func; + mutable KernelMutex lock; + + // Separated from make to improve inlining. + void make_body(bool block) const { + if (block) { + lock.Acquire(); + } else if (!lock.Try()) { + return; + } + MAKE_SCOPE_GUARD([&]() { lock.Release(); }); + if (func == nullptr) return; + T* ptr = func(); + obj.reset(ptr); + std::atomic_thread_fence(std::memory_order_release); + func = nullptr; + } + + __forceinline void make(bool block) const { + if (!created()) { + make_body(block); + } + } + +}; + +} // namespace rocr + +#endif // HSA_RUNTIME_CORE_UTIL_LAZY_PTR_H_ diff --git a/util/lnx/os_linux.cpp b/util/lnx/os_linux.cpp new file mode 100644 index 0000000000..aecca6c0fd --- /dev/null +++ b/util/lnx/os_linux.cpp @@ -0,0 +1,771 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2014-2024, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +#ifdef __linux__ +#include "core/util/os.h" +#include "core/util/utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "core/inc/runtime.h" +#if defined(__i386__) || defined(__x86_64__) +#include +#endif + +namespace rocr { +namespace os { + +struct ThreadArgs { + void* entry_args; + ThreadEntry entry_function; +}; + +void* __stdcall ThreadTrampoline(void* arg) { + ThreadArgs* ar = (ThreadArgs*)arg; + ThreadEntry CallMe = ar->entry_function; + void* Data = ar->entry_args; + delete ar; + CallMe(Data); + return nullptr; +} + +// Thread container allows multiple waits and separate close (destroy). +class os_thread { + public: + explicit os_thread(ThreadEntry function, void* threadArgument, uint stackSize) + : thread(0), lock(nullptr), state(RUNNING) { + int err; + std::unique_ptr args(new ThreadArgs); + lock = CreateMutex(); + if (lock == nullptr) return; + + args->entry_args = threadArgument; + args->entry_function = function; + + pthread_attr_t attrib; + err = pthread_attr_init(&attrib); + if (err != 0) { + fprintf(stderr, "pthread_attr_init failed: %s\n", strerror(err)); + return; + } + + if (stackSize != 0) { + stackSize = Max(uint(PTHREAD_STACK_MIN), stackSize); + stackSize = AlignUp(stackSize, 4096); + err = pthread_attr_setstacksize(&attrib, stackSize); + if (err != 0) { + fprintf(stderr, "pthread_attr_setstacksize failed: %s\n", strerror(err)); + err = pthread_attr_destroy(&attrib); + if (err != 0) { + fprintf(stderr, "pthread_attr_destroy failed: %s\n", strerror(err)); + return; + } + } + } + + int cores = 0; + cpu_set_t* cpuset = nullptr; + + if (core::Runtime::runtime_singleton_->flag().override_cpu_affinity()) { + cores = get_nprocs_conf(); + cpuset = CPU_ALLOC(cores); + if (cpuset == nullptr) { + fprintf(stderr, "CPU_ALLOC failed: %s\n", strerror(errno)); + return; + } + CPU_ZERO_S(CPU_ALLOC_SIZE(cores), cpuset); + for (int i = 0; i < cores; i++) { + CPU_SET_S(i, CPU_ALLOC_SIZE(cores), cpuset); + } + err = pthread_attr_setaffinity_np(&attrib, CPU_ALLOC_SIZE(cores), cpuset); + CPU_FREE(cpuset); + if (err != 0) { + fprintf(stderr, "pthread_setaffinity_np failed: %s\n", strerror(err)); + return; + } + } + + err = pthread_create(&thread, &attrib, ThreadTrampoline, args.get()); + + // Probably a stack size error since system limits can be different from PTHREAD_STACK_MIN + // Attempt to grow the stack within reason. + if ((err == EINVAL) && stackSize != 0) { + while (stackSize < 20 * 1024 * 1024) { + stackSize *= 2; + err = pthread_attr_setstacksize(&attrib, stackSize); + if (err != 0) { + fprintf(stderr, "pthread_attr_setstacksize failed: %s\n", strerror(err)); + return; + } + err = pthread_create(&thread, &attrib, ThreadTrampoline, args.get()); + if (err != EINVAL) break; + debug_print("pthread_create returned EINVAL, doubling stack size\n"); + } + } + + if (err == 0) + args.release(); + else + thread = 0; + + err = pthread_attr_destroy(&attrib); + if (err != 0) { + fprintf(stderr, "pthread_attr_destroy failed: %s\n", strerror(err)); + } + } + + os_thread(os_thread&& rhs) { + thread = rhs.thread; + lock = rhs.lock; + state = int(rhs.state); + rhs.thread = 0; + rhs.lock = nullptr; + } + + os_thread(os_thread&) = delete; + + ~os_thread() { + if (lock != nullptr) DestroyMutex(lock); + if ((state == RUNNING) && (thread != 0)) { + int err = pthread_detach(thread); + if (err != 0) fprintf(stderr, "pthread_detach failed: %s\n", strerror(err)); + } + } + + bool Valid() { return (lock != nullptr) && (thread != 0); } + + bool Wait() { + if (state == FINISHED) return true; + AcquireMutex(lock); + if (state == FINISHED) { + ReleaseMutex(lock); + return true; + } + int err = pthread_join(thread, NULL); + bool success = (err == 0); + if (success) state = FINISHED; + ReleaseMutex(lock); + return success; + } + + private: + pthread_t thread; + Mutex lock; + std::atomic state; + enum { FINISHED = 0, RUNNING = 1 }; +}; + +static_assert(sizeof(LibHandle) == sizeof(void*), "OS abstraction size mismatch"); +static_assert(sizeof(Semaphore) == sizeof(sem_t*), "OS abstraction size mismatch"); +static_assert(sizeof(Mutex) == sizeof(pthread_mutex_t*), "OS abstraction size mismatch"); +static_assert(sizeof(SharedMutex) == sizeof(pthread_rwlock_t*), "OS abstraction size mismatch"); +static_assert(sizeof(Thread) == sizeof(os_thread*), "OS abstraction size mismatch"); + +LibHandle LoadLib(std::string filename) { + void* ret = dlopen(filename.c_str(), RTLD_LAZY); + if (ret == nullptr) debug_print("LoadLib(%s) failed: %s\n", filename.c_str(), dlerror()); + return *(LibHandle*)&ret; +} + +void* GetExportAddress(LibHandle lib, std::string export_name) { + void* ret = dlsym(*(void**)&lib, export_name.c_str()); + + // dlsym searches the given library and all the library's load dependencies. + // Remaining code limits symbol lookup to only the library handle given. + // This lookup pattern matches Windows. + if (ret == NULL) return ret; + + link_map* map; + int err = dlinfo(*(void**)&lib, RTLD_DI_LINKMAP, &map); + if (err == -1) { + fprintf(stderr, "dlinfo failed: %s\n", dlerror()); + return nullptr; + } + + Dl_info info; + err = dladdr(ret, &info); + if (err == 0) { + fprintf(stderr, "dladdr failed.\n"); + return nullptr; + } + + if (strcmp(info.dli_fname, map->l_name) == 0) return ret; + + return NULL; +} + +void CloseLib(LibHandle lib) { dlclose(*(void**)&lib); } + +/* + * @brief Look for a symbol called "HSA_AMD_TOOL_PRIORITY" across all loaded + * shared libraries, and if found, store the name of the library + * + * @param[in]: info A dl_phdr_info struct pointer, which contains information + * about library's load address, header, and name. + * + * @param[in]: size integer size of dl_phdr_info struct + * + * @param[out]: data copy of the data argument to dl_phdr_iterate call + * + * @retval:: Return 0 on Success. If callback returns a non-zero value, + * dl_iterate_phdr() will stop processing, even if there are unprocessed + * shared objects. + */ + +static int callback(struct dl_phdr_info* info, size_t size, void* data) { + std::vector* loadedToolsLib = (std::vector*)data; + assert(loadedToolsLib != nullptr); + /* + * Check if lib name is not empty and its not a "vdso.so" lib, + * The vDSO is a special shared object file that is built into the Linux kernel. + * It is not a regular shared library and thus does not have all the properties + * of regular shared libraries. The way the vDSO is loaded and organized in memory + * is different from regular shared libraries and it's not guaranteed that it + * will have a specific segment or section. Hence its skipped. + */ + + if ((info) && (info->dlpi_name[0] != '\0')) { + if (std::string(info->dlpi_name).find("vdso.so") != std::string::npos) return 0; + + /* + * Iterate through the program headers of the loaded lib and check for PT_DYNAMIC program + * header. If the PT_DYNAMIC program header is found, use dlpi_addr and dlpi_phdr members + * of dl_phdr_info struct to get the address of the dynamic section of the loaded + * library in memory + */ + + for (int i = 0; i < info->dlpi_phnum; i++) { + if (info->dlpi_phdr[i].p_type == PT_DYNAMIC) { + Elf64_Dyn* dyn_section = (Elf64_Dyn*)(info->dlpi_addr + info->dlpi_phdr[i].p_vaddr); + + char* strings = nullptr; + Elf64_Xword limit = 0; + + /* + * The dynamic section is searched for DT_STRTAB (address of string table), + * and DT_STRSZ (size of string table) + * DT_NULL - Marks the end of the _DYNAMIC array + */ + + for (int j = 0;; j++) { + if (dyn_section[j].d_tag == DT_NULL) break; + + if (dyn_section[j].d_tag == DT_STRTAB) strings = (char*)(dyn_section[j].d_un.d_ptr); + + if (dyn_section[j].d_tag == DT_STRSZ) limit = dyn_section[j].d_un.d_val; + } + + if (strings == nullptr) debug_print("String table not found"); + + /* + * Hacky lookup, if string and symbol tables are found, + * iterate through the strings in string table and check if + * any string matches "HSA_AMD_TOOL_PRIORITY". + * If yes, then add the name of the library to the vector of + * lib names + */ + if (strings != nullptr) { + char* end = strings + limit; + while (strings < end) { + if (strcmp(strings, "HSA_AMD_TOOL_PRIORITY") == 0) { + loadedToolsLib->push_back(info->dlpi_name); + return 0; + } + strings += (strlen(strings) + 1); + } + } + } + } + } + return 0; +} + +std::vector GetLoadedToolsLib() { + std::vector ret; + std::vector names; + + /* Iterate through all of the loaded shared libraries in the process */ + dl_iterate_phdr(callback, &names); + + if (!names.empty()) { + for (auto& name : names) ret.push_back(LoadLib(name)); + } + + return ret; +} + +std::string GetLibraryName(LibHandle lib) { + link_map *map; + if(dlinfo(lib, RTLD_DI_LINKMAP, &map)!=0) + return ""; + return map->l_name; +} + +Semaphore CreateSemaphore() { + sem_t *sem = new sem_t; + sem_init(sem, 0, 0); + return *(Semaphore*)&sem; +} + +bool WaitSemaphore(Semaphore sem) { + while(sem_wait(*(sem_t**)&sem)) + if (errno != EINTR) return false; + + return true; +} + +void PostSemaphore(Semaphore sem) { + if (sem_post(*(sem_t**)&sem)) + assert(false && "Failed to post semaphore"); +} + +void DestroySemaphore(Semaphore sem) { + sem_destroy(*(sem_t**)&sem); + delete *(sem_t**)&sem; +} + +Mutex CreateMutex() { + pthread_mutex_t* mutex = new pthread_mutex_t; + pthread_mutex_init(mutex, NULL); + return *(Mutex*)&mutex; +} + +bool TryAcquireMutex(Mutex lock) { + return pthread_mutex_trylock(*(pthread_mutex_t**)&lock) == 0; +} + +bool AcquireMutex(Mutex lock) { + return pthread_mutex_lock(*(pthread_mutex_t**)&lock) == 0; +} + +void ReleaseMutex(Mutex lock) { + pthread_mutex_unlock(*(pthread_mutex_t**)&lock); +} + +void DestroyMutex(Mutex lock) { + pthread_mutex_destroy(*(pthread_mutex_t**)&lock); + delete *(pthread_mutex_t**)&lock; +} + +void Sleep(int delay_in_millisec) { usleep(delay_in_millisec * 1000); } + +void uSleep(int delayInUs) { usleep(delayInUs); } + +void YieldThread() { sched_yield(); } + +Thread CreateThread(ThreadEntry function, void* threadArgument, uint stackSize) { + os_thread* result = new os_thread(function, threadArgument, stackSize); + if (!result->Valid()) { + delete result; + return nullptr; + } + + return reinterpret_cast(result); +} + +void CloseThread(Thread thread) { delete reinterpret_cast(thread); } + +bool WaitForThread(Thread thread) { return reinterpret_cast(thread)->Wait(); } + +bool WaitForAllThreads(Thread* threads, uint threadCount) { + for (uint i = 0; i < threadCount; i++) WaitForThread(threads[i]); + return true; +} + +bool IsEnvVarSet(std::string env_var_name) { + char* buff = NULL; + buff = getenv(env_var_name.c_str()); + return (buff != NULL); +} + +void SetEnvVar(std::string env_var_name, std::string env_var_value) { + setenv(env_var_name.c_str(), env_var_value.c_str(), 1); +} + +int GetProcessId() { + return ::getpid(); +} + +std::string GetEnvVar(std::string env_var_name) { + char* buff; + buff = getenv(env_var_name.c_str()); + std::string ret; + if (buff) { + ret = buff; + } + return ret; +} + +size_t GetUserModeVirtualMemorySize() { +#ifdef _LP64 + // https://www.kernel.org/doc/Documentation/x86/x86_64/mm.txt : + // user space is 0000000000000000 - 00007fffffffffff (=47 bits) + return (size_t)(0x800000000000); +#else + return (size_t)(0xffffffff); // ~4GB +#endif +} + +size_t GetUsablePhysicalHostMemorySize() { + struct sysinfo info = {0}; + if (sysinfo(&info) != 0) { + return 0; + } + + const size_t physical_size = + static_cast(info.totalram * info.mem_unit); + return std::min(GetUserModeVirtualMemorySize(), physical_size); +} + +uintptr_t GetUserModeVirtualMemoryBase() { return (uintptr_t)0; } + +// Os event implementation +typedef struct EventDescriptor_ { + pthread_cond_t event; + pthread_mutex_t mutex; + bool state; + bool auto_reset; +} EventDescriptor; + +EventHandle CreateOsEvent(bool auto_reset, bool init_state) { + EventDescriptor* eventDescrp; + eventDescrp = (EventDescriptor*)malloc(sizeof(EventDescriptor)); + + pthread_mutex_init(&eventDescrp->mutex, NULL); + pthread_cond_init(&eventDescrp->event, NULL); + eventDescrp->auto_reset = auto_reset; + eventDescrp->state = init_state; + + EventHandle handle = reinterpret_cast(eventDescrp); + + return handle; +} + +int DestroyOsEvent(EventHandle event) { + if (event == NULL) { + return -1; + } + + EventDescriptor* eventDescrp = reinterpret_cast(event); + int ret_code = pthread_cond_destroy(&eventDescrp->event); + ret_code |= pthread_mutex_destroy(&eventDescrp->mutex); + free(eventDescrp); + return ret_code; +} + +int WaitForOsEvent(EventHandle event, unsigned int milli_seconds) { + if (event == NULL) { + return -1; + } + + EventDescriptor* eventDescrp = reinterpret_cast(event); + // Event wait time is 0 and state is non-signaled, return directly + if (milli_seconds == 0) { + int tmp_ret = pthread_mutex_trylock(&eventDescrp->mutex); + if (tmp_ret == EBUSY) { + // Timeout + return 1; + } + } + + int ret_code = 0; + pthread_mutex_lock(&eventDescrp->mutex); + if (!eventDescrp->state) { + if (milli_seconds == 0) { + ret_code = 1; + } else { + struct timespec ts; + struct timeval tp; + + ret_code = gettimeofday(&tp, NULL); + ts.tv_sec = tp.tv_sec; + ts.tv_nsec = tp.tv_usec * 1000; + + unsigned int sec = milli_seconds / 1000; + unsigned int mSec = milli_seconds % 1000; + + ts.tv_sec += sec; + ts.tv_nsec += mSec * 1000000; + + // More then one second, add 1 sec to the tv_sec elem + if (ts.tv_nsec > 1000000000) { + ts.tv_sec += 1; + ts.tv_nsec = ts.tv_nsec - 1000000000; + } + + ret_code = + pthread_cond_timedwait(&eventDescrp->event, &eventDescrp->mutex, &ts); + // Time out + if (ret_code == 110) { + ret_code = 0x14003; // 1 means time out in HSA + } + + if (ret_code == 0 && eventDescrp->auto_reset) { + eventDescrp->state = false; + } + } + } else if (eventDescrp->auto_reset) { + eventDescrp->state = false; + } + pthread_mutex_unlock(&eventDescrp->mutex); + + return ret_code; +} + +int SetOsEvent(EventHandle event) { + if (event == NULL) { + return -1; + } + + EventDescriptor* eventDescrp = reinterpret_cast(event); + int ret_code = 0; + ret_code = pthread_mutex_lock(&eventDescrp->mutex); + eventDescrp->state = true; + ret_code = pthread_mutex_unlock(&eventDescrp->mutex); + ret_code |= pthread_cond_signal(&eventDescrp->event); + + return ret_code; +} + +int ResetOsEvent(EventHandle event) { + if (event == NULL) { + return -1; + } + + EventDescriptor* eventDescrp = reinterpret_cast(event); + int ret_code = 0; + ret_code = pthread_mutex_lock(&eventDescrp->mutex); + eventDescrp->state = false; + ret_code = pthread_mutex_unlock(&eventDescrp->mutex); + + return ret_code; +} + +static double invPeriod = 0.0; + +uint64_t ReadAccurateClock() { + if (invPeriod == 0.0) AccurateClockFrequency(); + timespec time; + int err = clock_gettime(CLOCK_MONOTONIC_RAW, &time); + if (err != 0) { + perror("clock_gettime(CLOCK_MONOTONIC_RAW,...) failed"); + abort(); + } + return (uint64_t(time.tv_sec) * 1000000000ull + uint64_t(time.tv_nsec)) * invPeriod; +} + +uint64_t AccurateClockFrequency() { + static clockid_t clock = CLOCK_MONOTONIC; + static std::atomic first(true); + // Check kernel version - not a concurrency concern. + // use non-RAW for getres due to bug in older 2.6.x kernels + if (first.load(std::memory_order_acquire)) { + utsname kernelInfo; + if (uname(&kernelInfo) == 0) { + try { + std::string ver = kernelInfo.release; + size_t idx; + int major = std::stoi(ver, &idx); + int minor = std::stoi(ver.substr(idx + 1)); + if ((major >= 4) && (minor >= 4)) { + clock = CLOCK_MONOTONIC_RAW; + } + } catch (...) { + // Kernel version string doesn't conform to the standard pattern. + // Keep using the "safe" (non-RAW) clock. + } + } + first.store(false, std::memory_order_release); + } + timespec time; + int err = clock_getres(clock, &time); + if (err != 0) { + perror("clock_getres failed"); + abort(); + } + if (time.tv_sec != 0 || time.tv_nsec >= 0xFFFFFFFF) { + fprintf(stderr, + "clock_getres(CLOCK_MONOTONIC(_RAW),...) returned very low " + "frequency (<1Hz).\n"); + abort(); + } + if (invPeriod == 0.0) invPeriod = 1.0 / double(time.tv_nsec); + return 1000000000ull / uint64_t(time.tv_nsec); +} + +SharedMutex CreateSharedMutex() { + pthread_rwlockattr_t attrib; + int err = pthread_rwlockattr_init(&attrib); + if (err != 0) { + fprintf(stderr, "rw lock attribute init failed: %s\n", strerror(err)); + return nullptr; + } + +#ifdef __GLIBC__ + err = pthread_rwlockattr_setkind_np(&attrib, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP); + if (err != 0) { + fprintf(stderr, "Set rw lock attribute failure: %s\n", strerror(err)); + return nullptr; + } +#else + err = pthread_rwlockattr_setkind(&attrib, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP); + if (err != 0) { + fprintf(stderr, "Set rw lock attribute failure: %s\n", strerror(err)); + return nullptr; + } +#endif + + pthread_rwlock_t* lock = new pthread_rwlock_t; + err = pthread_rwlock_init(lock, &attrib); + if (err != 0) { + fprintf(stderr, "rw lock init failed: %s\n", strerror(err)); + return nullptr; + } + + pthread_rwlockattr_destroy(&attrib); + return lock; +} + +bool TryAcquireSharedMutex(SharedMutex lock) { + int err = pthread_rwlock_trywrlock(*(pthread_rwlock_t**)&lock); + return err == 0; +} + +bool AcquireSharedMutex(SharedMutex lock) { + int err = pthread_rwlock_wrlock(*(pthread_rwlock_t**)&lock); + return err == 0; +} + +void ReleaseSharedMutex(SharedMutex lock) { + int err = pthread_rwlock_unlock(*(pthread_rwlock_t**)&lock); + if (err != 0) { + fprintf(stderr, "SharedMutex unlock failed: %s\n", strerror(err)); + abort(); + } +} + +bool TrySharedAcquireSharedMutex(SharedMutex lock) { + int err = pthread_rwlock_tryrdlock(*(pthread_rwlock_t**)&lock); + return err == 0; +} + +bool SharedAcquireSharedMutex(SharedMutex lock) { + int err = pthread_rwlock_rdlock(*(pthread_rwlock_t**)&lock); + return err == 0; +} + +void SharedReleaseSharedMutex(SharedMutex lock) { + int err = pthread_rwlock_unlock(*(pthread_rwlock_t**)&lock); + if (err != 0) { + fprintf(stderr, "SharedMutex unlock failed: %s\n", strerror(err)); + abort(); + } +} + +void DestroySharedMutex(SharedMutex lock) { + pthread_rwlock_destroy(*(pthread_rwlock_t**)&lock); + delete *(pthread_rwlock_t**)&lock; +} + +static uint64_t sys_clock_period_ = 0; + +uint64_t ReadSystemClock() { + struct timespec ts; + clock_gettime(CLOCK_BOOTTIME, &ts); + uint64_t time = (uint64_t(ts.tv_sec) * 1000000000 + uint64_t(ts.tv_nsec)); + if (sys_clock_period_ != 1) + return time / sys_clock_period_; + else + return time; +} + +uint64_t SystemClockFrequency() { + struct timespec ts; + clock_getres(CLOCK_BOOTTIME, &ts); + sys_clock_period_ = (uint64_t(ts.tv_sec) * 1000000000 + uint64_t(ts.tv_nsec)); + return 1000000000 / sys_clock_period_; +} + +bool ParseCpuID(cpuid_t* cpuinfo) { +#if defined(__i386__) || defined(__x86_64__) + uint32_t eax, ebx, ecx, edx, max_eax = 0; + memset(cpuinfo, 0, sizeof(*cpuinfo)); + + /* Make sure current CPU supports at least EAX 4 */ + if (!__get_cpuid_max(0x80000004, NULL)) return false; + + // Manufacturer ID is a twelve-character ASCII string stored in order EBX, EDX, ECX. + if (!__get_cpuid(0, &max_eax, (uint32_t*)&cpuinfo->ManufacturerID[0], + (uint32_t*)&cpuinfo->ManufacturerID[8], + (uint32_t*)&cpuinfo->ManufacturerID[4])) { + return false; + } + + if (!strcmp(cpuinfo->ManufacturerID, "AuthenticAMD")) { + if (__get_cpuid(0x80000001, &eax, &ebx, &ecx, &edx)) { + cpuinfo->mwaitx = !!((ecx >> 29) & 0x1); + } + } + return true; +#else + return false; +#endif +} + +} // namespace os +} // namespace rocr + +#endif diff --git a/util/locks.h b/util/locks.h new file mode 100644 index 0000000000..6c0de49a07 --- /dev/null +++ b/util/locks.h @@ -0,0 +1,290 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +// Library of syncronization primitives - to be added to as needed. + +#ifndef HSA_RUNTIME_CORE_UTIL_LOCKS_H_ +#define HSA_RUNTIME_CORE_UTIL_LOCKS_H_ + +#include "utils.h" +#include "os.h" + +namespace rocr { + +class HybridMutex { + public: + HybridMutex():lock_(0) { + sem_ = os::CreateSemaphore(); + } + + ~HybridMutex() { + os::DestroySemaphore(sem_); + } + + bool Try() { + int old = 0; + return lock_.compare_exchange_strong(old, 1); + } + + bool Acquire() { + int cnt = maxSpinIterPause + maxSpinIterYield; + + int old = 0; + while (!lock_.compare_exchange_strong(old, 1)) { + cnt--; + if (cnt > maxSpinIterPause) { + _mm_pause(); + } else if (cnt-- > maxSpinIterYield) { + os::YieldThread(); + } else { + os::WaitSemaphore(sem_); + cnt = maxSpinIterPause + maxSpinIterYield; + } + old = 0; + } + return true; + } + + void Release() { + int old = 1; + if (lock_.compare_exchange_strong(old, 0)) + os::PostSemaphore(sem_); + } + + private: + std::atomic lock_; + os::Semaphore sem_; + const uint32_t maxSpinIterPause = 55; + const uint32_t maxSpinIterYield = 55; + + /// @brief: Disable copiable and assignable ability. + DISALLOW_COPY_AND_ASSIGN(HybridMutex); +}; + + +/// @brief: a class represents a kernel mutex. +/// Uses the kernel's scheduler to keep the waiting thread from being scheduled +/// until the lock is released (Best for long waits, though anything using +/// a kernel object is a long wait). +class KernelMutex { + public: + KernelMutex() { lock_ = os::CreateMutex(); } + ~KernelMutex() { os::DestroyMutex(lock_); } + + bool Try() { return os::TryAcquireMutex(lock_); } + bool Acquire() { return os::AcquireMutex(lock_); } + void Release() { os::ReleaseMutex(lock_); } + + private: + os::Mutex lock_; + + /// @brief: Disable copiable and assignable ability. + DISALLOW_COPY_AND_ASSIGN(KernelMutex); +}; + +/// @brief: represents a spin lock. +/// For very short hold durations on the order of the thread scheduling +/// quanta or less. +class SpinMutex { + public: + SpinMutex() { lock_ = 0; } + + bool Try() { + int old = 0; + return lock_.compare_exchange_strong(old, 1); + } + bool Acquire() { + int old = 0; + while (!lock_.compare_exchange_strong(old, 1)) + { + old=0; + os::YieldThread(); + } + return true; + } + void Release() { lock_ = 0; } + + private: + std::atomic lock_; + + /// @brief: Disable copiable and assignable ability. + DISALLOW_COPY_AND_ASSIGN(SpinMutex); +}; + +class KernelEvent { + public: + KernelEvent() { evt_ = os::CreateOsEvent(true, true); } + ~KernelEvent() { os::DestroyOsEvent(evt_); } + + bool IsSet() { return os::WaitForOsEvent(evt_, 0)==0; } + bool WaitForSet() { return os::WaitForOsEvent(evt_, 0xFFFFFFFF)==0; } + void Set() { os::SetOsEvent(evt_); } + void Reset() { os::ResetOsEvent(evt_); } + + private: + os::EventHandle evt_; + + /// @brief: Disable copiable and assignable ability. + DISALLOW_COPY_AND_ASSIGN(KernelEvent); +}; + +/// @brief: represents a yielding shared mutex. +/// aka read/write mutex +class KernelSharedMutex { + public: + /// @brief: Interfaces ScopedAcquire to shared operations. + class Shared { + public: + explicit Shared(KernelSharedMutex* lock) : lock_(lock) {} + bool Try() { return lock_->TryShared(); } + bool Acquire() { return lock_->AcquireShared(); } + void Release() { lock_->ReleaseShared(); } + + private: + KernelSharedMutex* lock_; + }; + + KernelSharedMutex() { lock_ = os::CreateSharedMutex(); } + ~KernelSharedMutex() { os::DestroySharedMutex(lock_); } + + // Exclusive mode operations + bool Try() { return os::TryAcquireSharedMutex(lock_); } + bool Acquire() { return os::AcquireSharedMutex(lock_); } + void Release() { os::ReleaseSharedMutex(lock_); } + + // Shared mode operations + bool TryShared() { return os::TrySharedAcquireSharedMutex(lock_); } + bool AcquireShared() { return os::SharedAcquireSharedMutex(lock_); } + void ReleaseShared() { os::SharedReleaseSharedMutex(lock_); } + + // Return shared operations interface + Shared shared() { return Shared(this); } + + private: + os::SharedMutex lock_; + + /// @brief: Disable copiable and assignable ability. + DISALLOW_COPY_AND_ASSIGN(KernelSharedMutex); +}; + +/// @brief: Type trait to identify mutex types +template class isMutex { + public: + enum { value = false }; +}; +template <> class isMutex { + public: + enum { value = true }; +}; +template <> class isMutex { + public: + enum { value = true }; +}; +template <> class isMutex { + public: + enum { value = true }; +}; +template <> class isMutex { + public: + enum { value = true }; +}; + +/// @brief: A class behaves as a lock in a scope. When trying to enter into the +/// critical section, creat a object of this class. After the control path goes +/// out of the scope, it will release the lock automatically. +template class ScopedAcquire { + public: + /// @brief: When constructing, acquire the lock. + /// @param: lock(Input), pointer to an existing lock. + explicit ScopedAcquire(LockType* lock) : lock_(lock), doRelease(true) { + static_assert(isMutex::value, "ScopedAcquire requires a mutex type."); + lock_.Acquire(); + } + explicit ScopedAcquire(LockType lock) : lock_(lock), doRelease(true) { + static_assert(!isMutex::value, "Mutex types are not copyable."); + lock_.Acquire(); + } + + /// @brief: when destructing, release the lock. + ~ScopedAcquire() { + if (doRelease) lock_.Release(); + } + + /// @brief: Release the lock early. Avoid using when possible. + void Release() { + lock_.Release(); + doRelease = false; + } + + private: + /// @brief: Adapts between pointers to mutex types and mutex pointer types. + template class container { + public: + container(T* lock) : lock_(lock) {} + __forceinline bool Acquire() { return lock_->Acquire(); } + __forceinline void Release() { return lock_->Release(); } + + private: + T* lock_; + }; + + /// @brief: Specialization for mutex pointer types. + template class container { + public: + container(T lock) : lock_(lock) {} + __forceinline bool Acquire() { return lock_.Acquire(); } + __forceinline void Release() { return lock_.Release(); } + + private: + T lock_; + }; + + container::value> lock_; + bool doRelease; + + /// @brief: Disable copiable and assignable ability. + DISALLOW_COPY_AND_ASSIGN(ScopedAcquire); +}; + +} // namespace rocr + +#endif // HSA_RUNTIME_CORE_SUTIL_LOCKS_H_ diff --git a/util/os.h b/util/os.h new file mode 100644 index 0000000000..2eec51a34e --- /dev/null +++ b/util/os.h @@ -0,0 +1,327 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2014-2024, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +// Minimal operating system abstraction interfaces. + +#ifndef HSA_RUNTIME_CORE_UTIL_OS_H_ +#define HSA_RUNTIME_CORE_UTIL_OS_H_ + +#include +#include +#include "utils.h" + +namespace rocr { +namespace os { +typedef void* LibHandle; +typedef void* Semaphore; +typedef void* Mutex; +typedef void* SharedMutex; +typedef void* Thread; +typedef void* EventHandle; + +enum class os_t { OS_WIN = 0, OS_LINUX, COUNT }; +static __forceinline std::underlying_type::type os_index(os_t val) { + return std::underlying_type::type(val); +} + +#ifdef _WIN32 +static const os_t current_os = os_t::OS_WIN; +#elif __linux__ +static const os_t current_os = os_t::OS_LINUX; +#else +static_assert(false, "Operating System not detected!"); +#endif + +/// @brief: Loads dynamic library based on file name. Return value will be NULL +/// if failed. +/// @param: filename(Input), file name of the library. +/// @return: LibHandle. +LibHandle LoadLib(std::string filename); + +/// @brief: Gets the address of exported symbol. Return NULl if failed. +/// @param: lib(Input), library handle which exporting from. +/// @param: export_name(Input), the name of the exported symbol. +/// @return: void*. +void* GetExportAddress(LibHandle lib, std::string export_name); + +/// @brief: Unloads the dynamic library. +/// @param: lib(Input), library handle which will be unloaded. +void CloseLib(LibHandle lib); + +/// @brief: Lists loaded tool libraries that contain +/// symbol HSA_AMD_TOOL_PRIORITY +/// @return: List of library handles +std::vector GetLoadedToolsLib(); + +/// @brief: Returns the library's path name. +/// @param: lib(Input), libray handle +/// @return: Path name of library +std::string GetLibraryName(LibHandle lib); + +/// @brief: Creates a Semaphore, will return NULL if failed. +/// @param: void. +/// @return: Semaphore. +Semaphore CreateSemaphore(); + +/// @brief: Waits for the semaphore. This is a blocking wait. +/// If the Semaphore is signalled, this function will return. +/// @param: sem(Input), handle to the semaphore. +/// @return: void. +bool WaitSemaphore(Semaphore sem); + +/// @brief: Post/Signal/Wake-up the semaphore +/// @param: sem(Input), handle to the semaphore. +/// @return: void. +void PostSemaphore(Semaphore sem); + +/// @brief: Destroys the semaphore. +/// @param: sem(Input), handle to the semaphore. +/// @return: void. +void DestroySemaphore(Semaphore sem); + +/// @brief: Creates a mutex, will return NULL if failed. +/// @param: void. +/// @return: Mutex. +Mutex CreateMutex(); + +/// @brief: Tries to acquire the mutex once, if successed, return true. +/// @param: lock(Input), handle to the mutex. +/// @return: bool. +bool TryAcquireMutex(Mutex lock); + +/// @brief: Aquires the mutex, if the mutex is locked, it will wait until it is +/// released. If the mutex is acquired successfully, it will return true. +/// @param: lock(Input), handle to the mutex. +/// @return: bool. +bool AcquireMutex(Mutex lock); + +/// @brief: Releases the mutex. +/// @param: lock(Input), handle to the mutex. +/// @return: void. +void ReleaseMutex(Mutex lock); + +/// @brief: Destroys the mutex. +/// @param: lock(Input), handle to the mutex. +/// @return: void. +void DestroyMutex(Mutex lock); + +/// @brief: Creates a shared mutex, will return NULL if failed. +/// @param: void. +/// @return: SharedMutex. +SharedMutex CreateSharedMutex(); + +/// @brief: Tries to acquire the mutex in exclusive mode once, if successed, return true. +/// @param: lock(Input), handle to the shared mutex. +/// @return: bool. +bool TryAcquireSharedMutex(SharedMutex lock); + +/// @brief: Aquires the mutex in exclusive mode, if the mutex is locked, it will wait until it is +/// released. If the mutex is acquired successfully, it will return true. +/// @param: lock(Input), handle to the mutex. +/// @return: bool. +bool AcquireSharedMutex(SharedMutex lock); + +/// @brief: Releases the mutex from exclusive mode. +/// @param: lock(Input), handle to the mutex. +/// @return: void. +void ReleaseSharedMutex(SharedMutex lock); + +/// @brief: Tries to acquire the mutex in shared mode once, if successed, return true. +/// @param: lock(Input), handle to the mutex. +/// @return: bool. +bool TrySharedAcquireSharedMutex(SharedMutex lock); + +/// @brief: Aquires the mutex in shared mode, if the mutex in exclusive mode, it will wait until it +/// is released. If the mutex is acquired successfully, it will return true. +/// @param: lock(Input), handle to the mutex. +/// @return: bool. +bool SharedAcquireSharedMutex(SharedMutex lock); + +/// @brief: Releases the mutex from shared mode. +/// @param: lock(Input), handle to the mutex. +/// @return: void. +void SharedReleaseSharedMutex(SharedMutex lock); + +/// @brief: Destroys the mutex. +/// @param: lock(Input), handle to the mutex. +/// @return: void. +void DestroySharedMutex(SharedMutex lock); + +/// @brief: Puts current thread to sleep. +/// @param: delayInMs(Input), time in millisecond for sleeping. +/// @return: void. +void Sleep(int delayInMs); + +/// @brief: Puts current thread to sleep. +/// @param: delayInMs(Input), time in millisecond for sleeping. +/// @return: void. +void uSleep(int delayInUs); + +/// @brief: Yields current thread. +/// @param: void. +/// @return: void. +void YieldThread(); + +typedef void (*ThreadEntry)(void*); + +/// @brief: Creates a thread will return NULL if failed. +/// @param: entry_function(Input), a pointer to the function which the thread +/// starts from. +/// @param: entry_argument(Input), a pointer to the argument of the thread +/// function. +/// @param: stack_size(Input), size of the thread's stack, 0 by default. +/// @return: Thread, a handle to thread created. +Thread CreateThread(ThreadEntry entry_function, void* entry_argument, + uint stack_size = 0); + +/// @brief: Destroys the thread. +/// @param: thread(Input), thread handle to what will be destroyed. +/// @return: void. +void CloseThread(Thread thread); + +/// @brief: Waits for specific thread to finish, if successful, return true. +/// @param: thread(Input), handle to waiting thread. +/// @return: bool. +bool WaitForThread(Thread thread); + +/// @brief: Waits for multiple threads to finish, if successful, return true. +/// @param; threads(Input), a pointer to a list of thread handle. +/// @param: thread_count(Input), number of threads to be waited on. +/// @return: bool. +bool WaitForAllThreads(Thread* threads, uint thread_count); + +/// @brief: Determines if environment key is set. +/// @param: env_var_name(Input), name of the environment value. +/// @return: bool, true for binding any value to environment key, +/// including an empty string. False otherwise +bool IsEnvVarSet(std::string env_var_name); + +/// @brief: Sets the environment value. +/// @param: env_var_name(Input), name of the environment value. +/// @param: env_var_value(Input), value of the environment value.s +/// @return: void. +void SetEnvVar(std::string env_var_name, std::string env_var_value); + +/// @brief: Gets the value of environment value. +/// @param: env_var_name(Input), name of the environment value. +/// @return: std::string, value of the environment value, returned as string. +std::string GetEnvVar(std::string env_var_name); + +/// @brief: Gets the process ID. +/// @param: void +/// @return: int, process ID returned as int. +int GetProcessId(); + +/// @brief: Gets the max virtual memory size accessible to the application. +/// @param: void. +/// @return: size_t, size of the accessible memory to the application. +size_t GetUserModeVirtualMemorySize(); + +/// @brief: Gets the max physical host system memory size. +/// @param: void. +/// @return: size_t, size of the physical host system memory. +size_t GetUsablePhysicalHostMemorySize(); + +/// @brief: Gets the virtual memory base address. It is hardcoded to 0. +/// @param: void. +/// @return: uintptr_t, always 0. +uintptr_t GetUserModeVirtualMemoryBase(); + +/// @brief os event api, create an event +/// @param: auto_reset whether an event can reset the status automatically +/// @param: init_state initial state of the event +/// @return: event handle +EventHandle CreateOsEvent(bool auto_reset, bool init_state); + +/// @brief os event api, destroy an event +/// @param: event handle +/// @return: whether destroy is correct +int DestroyOsEvent(EventHandle event); + +/// @brief os event api, wait on event +/// @param: event Event handle +/// @param: milli_seconds wait time +/// @return: Indicate success or timeout +int WaitForOsEvent(EventHandle event, unsigned int milli_seconds); + +/// @brief os event api, set event state +/// @param: event Event handle +/// @return: Whether event set is correct +int SetOsEvent(EventHandle event); + +/// @brief os event api, reset event state +/// @param: event Event handle +/// @return: Whether event reset is correct +int ResetOsEvent(EventHandle event); + +/// @brief reads a clock which is deemed to be accurate for elapsed time +/// measurements, though not necessarilly fast to query +/// @return clock counter value +uint64_t ReadAccurateClock(); + +/// @brief retrieves the frequency in Hz of the unit used in ReadAccurateClock. +/// It does not necessarilly reflect the resolution of the clock, but is the +/// value needed to convert a difference in the clock's counter value to elapsed +/// seconds. This frequency does not change at runtime. +/// @return returns the frequency +uint64_t AccurateClockFrequency(); + +/// @brief read the system clock which serves as the HSA system clock +/// counter in KFD. +uint64_t ReadSystemClock(); + +/// @brief read the system clock frequency +uint64_t SystemClockFrequency(); + +typedef struct cpuid_s { + char ManufacturerID[13]; // 12 char, NULL terminated + bool mwaitx; +} cpuid_t; + +/// @brief parse CPUID +/// @param: cpuinfo struct to be filled +bool ParseCpuID(cpuid_t* cpuinfo); + +} // namespace os +} // namespace rocr + +#endif // HSA_RUNTIME_CORE_UTIL_OS_H_ diff --git a/util/simple_heap.h b/util/simple_heap.h new file mode 100644 index 0000000000..6c7822bcdb --- /dev/null +++ b/util/simple_heap.h @@ -0,0 +1,363 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +// A simple best fit memory allocator with eager compaction. Manages block sub-allocation. +// For use when memory efficiency is more important than allocation speed. +// O(log n) time. + +#ifndef HSA_RUNTME_CORE_UTIL_SIMPLE_HEAP_H_ +#define HSA_RUNTME_CORE_UTIL_SIMPLE_HEAP_H_ + +#include +#include +#include + +#include "core/util/utils.h" + +namespace rocr { + +template class SimpleHeap { + private: + struct Fragment_T { + typedef std::multimap::iterator ptr_t; + ptr_t free_list_entry_; + struct { + size_t size : 62; + bool discard : 1; + bool free : 1; + }; + + Fragment_T(ptr_t Iterator, size_t Len, bool Free) + : free_list_entry_(Iterator), size(Len), discard(false), free(Free) {} + Fragment_T() = default; + }; + + struct Block { + uintptr_t base_ptr_; + size_t length_; + + Block(uintptr_t base, size_t length) : base_ptr_(base), length_(length) {} + Block() = default; + }; + + Allocator block_allocator_; + + std::multimap free_list_; + std::map> block_list_; + std::deque block_cache_; + + // Size of blocks that are at least partially in use. + size_t in_use_size_; + // Total size of block cache + size_t cache_size_; + + __forceinline bool isFree(const Fragment_T& node) { return node.free; } + __forceinline void setUsed(Fragment_T& node) { + node.free = false; + node.free_list_entry_ = free_list_.end(); + } + __forceinline void setFree(Fragment_T& node, typename Fragment_T::ptr_t Iterator) { + node.free_list_entry_ = Iterator; + node.free = true; + } + __forceinline Fragment_T makeFragment(size_t Len) { + return Fragment_T(free_list_.end(), Len, false); + } + __forceinline Fragment_T makeFragment(typename Fragment_T::ptr_t Iterator, size_t Len) { + return Fragment_T(Iterator, Len, true); + } + __forceinline void removeFreeListEntry(Fragment_T& node) { + if (node.free_list_entry_ != free_list_.end()) { + free_list_.erase(node.free_list_entry_); + node.free_list_entry_ = free_list_.end(); + } + } + __forceinline void discard(Fragment_T& node) { + removeFreeListEntry(node); + node.discard = true; + } + + public: + explicit SimpleHeap(const Allocator& BlockAllocator = Allocator()) + : block_allocator_(BlockAllocator), in_use_size_(0), cache_size_(0) {} + ~SimpleHeap() { + trim(); + // Leak here may be due to the user. Check is for debugging only. + // assert(in_use_size_ == 0 && "Leak in SimpleHeap."); + } + + SimpleHeap(const SimpleHeap& rhs) = delete; + SimpleHeap(SimpleHeap&& rhs) = delete; + SimpleHeap& operator=(const SimpleHeap& rhs) = delete; + SimpleHeap& operator=(SimpleHeap&& rhs) = delete; + + void* alloc(size_t bytes) { + // Find best fit. + uintptr_t base; + size_t size; + // For bytes >= 2MB, the requested mem should be aligned + size_t align_bytes = bytes; + const int retry = bytes >= GPU_HUGE_PAGE_SIZE ? 1 : 0; + size_t align = bytes >= GPU_HUGE_PAGE_SIZE ? GPU_HUGE_PAGE_SIZE : DEFAULT_GPU_PAGE_SIZE; + + for (int i = 0; i <= retry; i++) { + auto free_fragment = free_list_.lower_bound(align_bytes); + if (free_fragment == free_list_.end()) break; + + uintptr_t addr = free_fragment->second; + size = free_fragment->first; + + assert(size >= bytes && "SimpleHeap: map lower_bound failure."); + + // Find the containing block and fragment + auto it = block_list_.upper_bound(addr); + it--; + auto& frag_map = it->second; + const auto& fragment = frag_map.find(addr); + + assert(fragment != frag_map.end() && "Inconsistency in SimpleHeap."); + assert(size == fragment->second.size && "Inconsistency in SimpleHeap."); + + size_t delta = addr & (align - 1); + if (!delta) { + // already find aligned address + base = addr; + free_list_.erase(free_fragment); + // Sub-allocate from fragment. + fragment->second.size = bytes; + setUsed(fragment->second); + // Record remaining free space. + if (size > bytes) { + free_fragment = free_list_.insert(std::make_pair(size - bytes, base + bytes)); + frag_map[base + bytes] = makeFragment(free_fragment, size - bytes); + } + } else { + // If this is the first request and the requested size is not enough for alignment, + // then request for a bigger hole and do trim. + if (i == 0 && size < bytes + align - delta) { + align_bytes += align; + continue; + } + + uintptr_t aligned_base = addr + align - delta; + base = aligned_base; + + // Erase the old free list + free_list_.erase(free_fragment); + + // fragment 1 - free + free_fragment = free_list_.insert(std::make_pair(aligned_base - addr, addr)); + frag_map[addr] = makeFragment(free_fragment, aligned_base - addr); + + //fragment 2 - used + frag_map[base] = makeFragment(bytes); + + // fragement 3 - free + if (size > aligned_base - addr + bytes) { + free_fragment = free_list_.insert(std::make_pair(size - (aligned_base - addr) - bytes, aligned_base + bytes)); + frag_map[aligned_base + bytes] = makeFragment(free_fragment, size - (aligned_base - addr) - bytes); + } + } + return reinterpret_cast(base); + } + + // No usable fragment, check block cache + if (bytes < default_block_size() && !block_cache_.empty()) { + const auto& block = block_cache_.back(); + base = block.base_ptr_; + size = block.length_; + block_cache_.pop_back(); + cache_size_ -= size; + } else { // Alloc new block - new block may be larger than default. + void* ptr = block_allocator_.alloc(bytes, size); + base = reinterpret_cast(ptr); + assert(ptr != nullptr && "Block allocation failed, Allocator is expected to throw."); + } + + in_use_size_ += size; + assert(size >= bytes && "Alloc exceeds block size."); + // Sub alloc and insert free region. + if (size > bytes) { + auto free_fragment = free_list_.insert(std::make_pair(size - bytes, base + bytes)); + block_list_[base][base + bytes] = makeFragment(free_fragment, size - bytes); + } + // Track used region + block_list_[base][base] = makeFragment(bytes); + + // Disallow multiple suballocation from large blocks. + // Prevents a small allocation from retaining a large block. + if (bytes > default_block_size()) { + bool err = discardBlock(reinterpret_cast(base)); + assert(err && "Large block discard failed."); + } + + return reinterpret_cast(base); + } + + bool free(void* ptr) { + if (ptr == nullptr) return true; + + uintptr_t base = reinterpret_cast(ptr); + + // Find fragment and validate. + auto frag_map_it = block_list_.upper_bound(base); + if (frag_map_it == block_list_.begin()) return false; + frag_map_it--; + auto& frag_map = frag_map_it->second; + auto fragment = frag_map.find(base); + if (fragment == frag_map.end() || isFree(fragment->second)) return false; + + bool discard = fragment->second.discard; + + // Merge lower + if (fragment != frag_map.begin()) { + auto lower = fragment; + lower--; + if (isFree(lower->second)) { + removeFreeListEntry(lower->second); + lower->second.size += fragment->second.size; + frag_map.erase(fragment); + fragment = lower; + } + } + + // Merge upper + { + auto upper = fragment; + upper++; + if ((upper != frag_map.end()) && isFree(upper->second)) { + removeFreeListEntry(upper->second); + fragment->second.size += upper->second.size; + frag_map.erase(upper); + } + } + + // Release whole free blocks. + if (frag_map.size() == 1) { + Block block(fragment->first, fragment->second.size); + block_list_.erase(frag_map_it); + + // Discard or add to the block cache. + if (discard) { + block_allocator_.free(reinterpret_cast(block.base_ptr_), block.length_); + } else { + block_cache_.push_back(block); + cache_size_ += block.length_; + in_use_size_ -= block.length_; + } + + balance(); + + // Don't publish free space since block was moved to the cache. + return true; + } + + // Don't report free memory if discarding the fragment. + if (discard) return true; + + // Report free fragment + const auto& freeEntry = + free_list_.insert(std::make_pair(size_t(fragment->second.size), fragment->first)); + setFree(fragment->second, freeEntry); + + return true; + } + + void balance() { + // Release old blocks when over cache limit. + while ((block_cache_.size() > 1) && (cache_size_ > in_use_size_ * 2)) { + const auto& block = block_cache_.front(); + block_allocator_.free(reinterpret_cast(block.base_ptr_), block.length_); + cache_size_ -= block.length_; + block_cache_.pop_front(); + } + } + + void trim() { + for (const auto& block : block_cache_) + block_allocator_.free(reinterpret_cast(block.base_ptr_), block.length_); + block_cache_.clear(); + cache_size_ = 0; + } + + size_t cache_size() const { return cache_size_; } + + size_t default_block_size() const { return block_allocator_.block_size(); } + + // Prevent reuse of the block containing ptr. No further fragments will be allocated from the + // block and the block will not be added to the block cache when it is free. + bool discardBlock(void* ptr) { + if (ptr == nullptr) return true; + + uintptr_t base = reinterpret_cast(ptr); + + // Find block validate. + auto frag_map_it = block_list_.upper_bound(base); + if (frag_map_it == block_list_.begin()) return false; + frag_map_it--; + auto& frag_map = frag_map_it->second; + if ((base < frag_map.begin()->first) || + (frag_map.rbegin()->first + frag_map.rbegin()->second.size <= base)) + return false; + + // Is block already discarded? + if (frag_map.begin()->second.discard) return true; + + // Mark all fragments for discard and compute block size. Removes freelist records for all + // fragments in the block. + size_t size = 0; + for (auto& frag : frag_map) { + discard(frag.second); + size += frag.second.size; + } + + // Remove discarded block from in-use tracking and rebalance the block cache. + in_use_size_ -= size; + balance(); + + return true; + } +}; + +} // namespace rocr + +#endif // HSA_RUNTME_CORE_UTIL_SIMPLE_HEAP_H_ diff --git a/util/small_heap.cpp b/util/small_heap.cpp new file mode 100644 index 0000000000..9fe5da5fab --- /dev/null +++ b/util/small_heap.cpp @@ -0,0 +1,185 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +#include "small_heap.h" + +namespace rocr { + +// Inserts node into freelist after place. +// Assumes node will not be an end of the list (list has guard nodes). +void SmallHeap::insertafter(SmallHeap::iterator_t place, SmallHeap::iterator_t node) { + assert(place->first < node->first && "Order violation"); + assert(isfree(place->second) && "Freelist operation error."); + iterator_t next = place->second.next; + node->second.next = next; + node->second.prior = place; + place->second.next = node; + next->second.prior = node; +} + +// Removes node from freelist. +// Assumes node will not be an end of the list (list has guard nodes). +void SmallHeap::remove(SmallHeap::iterator_t node) { + assert(isfree(node->second) && "Freelist operation error."); + node->second.prior->second.next = node->second.next; + node->second.next->second.prior = node->second.prior; + setused(node->second); +} + +// Returns high if merge failed or the merged node. +SmallHeap::memory_t::iterator SmallHeap::merge(SmallHeap::memory_t::iterator low, + SmallHeap::memory_t::iterator high) { + assert(isfree(low->second) && "Merge with allocated block"); + assert(isfree(high->second) && "Merge with allocated block"); + + if ((char*)low->first + low->second.len != (char*)high->first) return high; + + assert(!islastfree(high->second) && "Illegal merge."); + + low->second.len += high->second.len; + low->second.next = high->second.next; + high->second.next->second.prior = low; + + memory.erase(high); + return low; +} + +void SmallHeap::free(void* ptr) { + if (ptr == nullptr) return; + + auto iterator = memory.find(ptr); + + // Check for illegal free + if (iterator == memory.end()) { + assert(false && "Illegal free."); + return; + } + + // Return memory to total and link node into free list + total_free += iterator->second.len; + + // Could also traverse the free list which might be faster in some cases. + auto before = iterator; + before--; + while (!isfree(before->second)) before--; + assert(before->second.next->first > iterator->first && "Inconsistency in small heap."); + insertafter(before, iterator); + + // Attempt compaction + iterator = merge(before, iterator); + merge(iterator, iterator->second.next); + + // Update lowHighBondary + high.erase(ptr); +} + +void* SmallHeap::alloc(size_t bytes) { + // Is enough memory available? + if ((bytes > total_free) || (bytes == 0)) return nullptr; + + iterator_t current; + + // Walk the free list and allocate at first fitting location + current = firstfree(); + while (!islastfree(current->second)) { + if (bytes <= current->second.len) { + // Decrement from total + total_free -= bytes; + + // Split node + if (bytes != current->second.len) { + void* remaining = (char*)current->first + bytes; + Node& node = memory[remaining]; + node.len = current->second.len - bytes; + current->second.len = bytes; + insertafter(current, memory.find(remaining)); + } + + remove(current); + return current->first; + } + current = current->second.next; + } + assert(current->second.len == 0 && "Freelist corruption."); + + // Can't service the request due to fragmentation + return nullptr; +} + +void* SmallHeap::alloc_high(size_t bytes) { + // Is enough memory available? + if ((bytes > total_free) || (bytes == 0)) return nullptr; + + iterator_t current; + + // Walk the free list and allocate at first fitting location + current = lastfree(); + while (!isfirstfree(current->second)) { + if (bytes <= current->second.len) { + // Decrement from total + total_free -= bytes; + + void* alloc; + // Split node + if (bytes != current->second.len) { + alloc = (char*)current->first + current->second.len - bytes; + current->second.len -= bytes; + Node& node = memory[alloc]; + node.len = bytes; + setused(node); + } else { + alloc = current->first; + remove(current); + } + + high.insert(alloc); + return alloc; + } + current = current->second.prior; + } + assert(current->second.len == 0 && "Freelist corruption."); + + // Can't service the request due to fragmentation + return nullptr; +} + +} // namespace rocr diff --git a/util/small_heap.h b/util/small_heap.h new file mode 100644 index 0000000000..e1f5d7bdeb --- /dev/null +++ b/util/small_heap.h @@ -0,0 +1,131 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +// A simple first fit memory allocator with eager compaction. For use with few +// items (where list iteration is faster than trees). +// Not thread safe! + +#ifndef HSA_RUNTME_CORE_UTIL_SMALL_HEAP_H_ +#define HSA_RUNTME_CORE_UTIL_SMALL_HEAP_H_ + +#include +#include + +#include "utils.h" + +namespace rocr { + +class SmallHeap { + private: + struct Node; + typedef std::map memory_t; + typedef memory_t::iterator iterator_t; + + struct Node { + size_t len; + iterator_t next; + iterator_t prior; + }; + + SmallHeap(const SmallHeap& rhs) = delete; + SmallHeap& operator=(const SmallHeap& rhs) = delete; + + void* const pool; + const size_t length; + + size_t total_free; + memory_t memory; + std::set high; + + __forceinline bool isfree(const Node& node) const { return node.next != memory.begin(); } + __forceinline bool islastfree(const Node& node) const { return node.next == memory.end(); } + __forceinline bool isfirstfree(const Node& node) const { return node.prior == memory.end(); } + __forceinline void setlastfree(Node& node) { node.next = memory.end(); } + __forceinline void setfirstfree(Node& node) { node.prior = memory.end(); } + __forceinline void setused(Node& node) { node.next = memory.begin(); } + + __forceinline iterator_t firstfree() { return memory.begin()->second.next; } + __forceinline iterator_t lastfree() { return memory.rbegin()->second.prior; } + void insertafter(iterator_t place, iterator_t node); + void remove(iterator_t node); + iterator_t merge(iterator_t low, iterator_t high); + + public: + SmallHeap() : pool(nullptr), length(0), total_free(0) {} + SmallHeap(void* base, size_t length) + : pool(base), length(length), total_free(length) { + assert(pool != nullptr && "Invalid base address."); + assert(pool != (void*)0xFFFFFFFFFFFFFFFFull && "Invalid base address."); + assert((char*)pool + length != (char*)0xFFFFFFFFFFFFFFFFull && "Invalid pool bounds."); + + Node& start = memory[0]; + Node& node = memory[pool]; + Node& end = memory[(void*)0xFFFFFFFFFFFFFFFFull]; + + start.len = 0; + start.next = memory.find(pool); + setfirstfree(start); + + node.len = length; + node.prior = memory.begin(); + node.next = --memory.end(); + + end.len = 0; + end.prior = start.next; + setlastfree(end); + + high.insert((void*)0xFFFFFFFFFFFFFFFFull); + } + + void* alloc(size_t bytes); + void* alloc_high(size_t bytes); + void free(void* ptr); + + void* base() const { return pool; } + size_t size() const { return length; } + size_t remaining() const { return total_free; } + void* high_split() const { return *high.begin(); } +}; + +} // namespace rocr + +#endif diff --git a/util/timer.cpp b/util/timer.cpp new file mode 100644 index 0000000000..5419e4f61d --- /dev/null +++ b/util/timer.cpp @@ -0,0 +1,111 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +#include "core/util/timer.h" + +namespace rocr { +namespace timer { + +accurate_clock::init::init() { + freq = os::AccurateClockFrequency(); + accurate_clock::period_ns = 1e9 / double(freq); +} + +// Calibrates the fast clock using the accurate clock. +fast_clock::init::init() { + typedef accurate_clock clock; + clock::duration delay(std::chrono::milliseconds(1)); + + // calibrate clock + fast_clock::raw_rep min = 0; + clock::duration elapsed; + + do { + elapsed = clock::duration::max(); + + for (int t = 0; t < 10; t++) { + fast_clock::raw_rep r1, r2; + clock::time_point t0, t1, t2, t3; + + t0 = clock::now(); + std::atomic_signal_fence(std::memory_order_acq_rel); + r1 = fast_clock::raw_now(); + std::atomic_signal_fence(std::memory_order_acq_rel); + t1 = clock::now(); + std::atomic_signal_fence(std::memory_order_acq_rel); + + do { + t2 = clock::now(); + } while (t2 - t1 < delay); + + std::atomic_signal_fence(std::memory_order_acq_rel); + r2 = fast_clock::raw_now(); + std::atomic_signal_fence(std::memory_order_acq_rel); + t3 = clock::now(); + + // If elapsed time is shorter than last recorded time and both the start + // and end times are confirmed correlated then record the clock readings. + // This protects against inaccuracy due to thread switching + if ((t3 - t1 < elapsed) && ((t1 - t0) * 10 < (t2 - t1)) && + ((t3 - t2) * 10 < (t2 - t1))) { + elapsed = t3 - t1; + min = r2 - r1; + } + } + delay += delay; + } while (min < 1000); + + fast_clock::freq = double(min) / duration_in_seconds(elapsed); + fast_clock::period_ps = 1e12 / fast_clock::freq; + // printf("Timer setup took %f ms\n", duration_in_seconds(elapsed)*1000.0f); + // printf("Fast clock frequency: %f MHz\n", double(fast_clock::freq)/1e6); +} + +double accurate_clock::period_ns; +accurate_clock::raw_frequency accurate_clock::freq; +accurate_clock::init accurate_clock::accurate_clock_init; + +double fast_clock::period_ps; +fast_clock::raw_frequency fast_clock::freq; +fast_clock::init fast_clock::fast_clock_init; +} // namespace timer +} // namespace rocr diff --git a/util/timer.h b/util/timer.h new file mode 100644 index 0000000000..155a11a393 --- /dev/null +++ b/util/timer.h @@ -0,0 +1,173 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +#ifndef HSA_RUNTIME_CORE_UTIL_TIMER_H_ +#define HSA_RUNTIME_CORE_UTIL_TIMER_H_ + +#include "core/util/utils.h" +#include "core/util/os.h" +#include +#include +#include + +namespace rocr { +namespace timer { + +// Needed to patch around a mixed arithmetic bug in MSVC's duration_cast as of +// VS 2013. +template +struct wide_type { + typedef double type; +}; +template <> +struct wide_type { + typedef uintmax_t type; +}; +template <> +struct wide_type { + typedef intmax_t type; +}; + +template +static __forceinline To + duration_cast(const std::chrono::duration& d) { + typedef typename wide_type::value, + std::is_signed::value>::type wide; + typedef std::chrono::duration unit_convert_t; + + unit_convert_t temp = std::chrono::duration_cast(d); + return To(static_cast(temp.count())); +} +// End patch + +template +static __forceinline double duration_in_seconds( + std::chrono::duration delta) { + typedef std::chrono::duration> seconds; + return seconds(delta).count(); +} + +template +static __forceinline rep duration_from_seconds(double delta) { + typedef std::chrono::duration> seconds; + return std::chrono::duration_cast(seconds(delta)); +} + +// Provices a C++11 standard clock interface to the os::AccurateClock functions +class accurate_clock { + public: + typedef double rep; + typedef std::nano period; + typedef std::chrono::duration duration; + typedef std::chrono::time_point time_point; + + static const bool is_steady = true; + + static __forceinline time_point now() { + return time_point(duration(raw_now() * period_ns)); + } + + // These two extra APIs and types let us use clocks without conversion to the + // arbitrary period unit + typedef uint64_t raw_rep; + typedef uint64_t raw_frequency; + + static __forceinline raw_rep raw_now() { return os::ReadAccurateClock(); } + static __forceinline raw_frequency raw_freq() { return freq; } + + private: + static double period_ns; + static raw_frequency freq; + + class init { + public: + init(); + }; + static init accurate_clock_init; +}; + +// Provices a C++11 standard clock interface to the lowest latency approximate +// clock +class fast_clock { + public: + typedef double rep; + typedef std::pico period; + typedef std::chrono::duration duration; + typedef std::chrono::time_point time_point; + + static const bool is_steady = true; + + static __forceinline time_point now() { + return time_point(duration(raw_now() * period_ps)); + } + + // These two extra APIs and types let us use clocks without conversion to the + // arbitrary period unit + typedef uint64_t raw_rep; + typedef double raw_frequency; + +#if defined(__x86_64__) || defined(_M_X64) + static __forceinline raw_rep raw_now() { return __rdtsc(); } + static __forceinline raw_frequency raw_freq() { return freq; } +#else + static __forceinline raw_rep raw_now() { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC_RAW, &ts); + return (raw_rep(ts.tv_sec) * 1000000000 + raw_rep(ts.tv_nsec)); + } + static __forceinline raw_frequency raw_freq() { return 1.e-9; } +#endif + + private: + static double period_ps; + static raw_frequency freq; + + class init { + public: + init(); + }; + static init fast_clock_init; +}; +} // namespace timer +} // namespace rocr + +#endif diff --git a/util/utils.h b/util/utils.h new file mode 100644 index 0000000000..c2fa8ef32c --- /dev/null +++ b/util/utils.h @@ -0,0 +1,424 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2014-2024, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +// Generally useful utility functions + +#ifndef HSA_RUNTIME_CORE_UTIL_UTILS_H_ +#define HSA_RUNTIME_CORE_UTIL_UTILS_H_ + +#include "stdint.h" +#include "stddef.h" +#include "stdlib.h" +#include "stdarg.h" +#include "unistd.h" +#include +#include +#include +#include +#include +#include + +namespace rocr { +extern FILE* log_file; +extern uint8_t log_flags[8]; + +typedef unsigned int uint; +typedef uint64_t uint64; + +#if defined(__GNUC__) +#if defined(__i386__) || defined(__x86_64__) +#include +#endif + +// 2MB huge page size +#define GPU_HUGE_PAGE_SIZE (2 << 20) + +// 4KB page size +#define DEFAULT_GPU_PAGE_SIZE (1 << 12) + +#define __forceinline __inline__ __attribute__((always_inline)) +#define __declspec(x) __attribute__((x)) +#undef __stdcall +#define __stdcall // __attribute__((__stdcall__)) +#define __ALIGNED__(x) __attribute__((aligned(x))) + +void log_printf(const char* file, int line, const char* format, ...); + +static __forceinline void* _aligned_malloc(size_t size, size_t alignment) { +#ifdef _ISOC11_SOURCE + return aligned_alloc(alignment, size); +#else + void *mem = NULL; + if (0 != posix_memalign(&mem, alignment, size)) return NULL; + return mem; +#endif +} +static __forceinline void _aligned_free(void* ptr) { return free(ptr); } +#elif defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64)) +#include "intrin.h" +#define __ALIGNED__(x) __declspec(align(x)) +#if (_MSC_VER < 1800) // < VS 2013 +static __forceinline unsigned long long int strtoull(const char* str, + char** endptr, int base) { + return static_cast(_strtoui64(str, endptr, base)); +} +#endif +#if (_MSC_VER < 1900) // < VS 2015 +#define thread_local __declspec(thread) +#endif +#else +#error "Compiler and/or processor not identified." +#endif + +#define STRING2(x) #x +#define STRING(x) STRING2(x) + +#define PASTE2(x, y) x##y +#define PASTE(x, y) PASTE2(x, y) + +#ifdef NDEBUG +#define debug_warning_n(exp, limit) \ + do { \ + } while (false) +#else +#define debug_warning_n(exp, limit) \ + do { \ + static std::atomic count(0); \ + if (!(exp) && (limit == 0 || count < limit)) { \ + fprintf(stderr, "Warning: " STRING(exp) " in %s, " __FILE__ ":" STRING(__LINE__) "\n", \ + __PRETTY_FUNCTION__); \ + count++; \ + } \ + } while (false) +#endif +#define debug_warning(exp) debug_warning_n((exp), 0) + +#ifdef NDEBUG +#define debug_print(fmt, ...) \ + do { \ + } while (false) +#else +#define debug_print(fmt, ...) \ + do { \ + fprintf(stderr, fmt, ##__VA_ARGS__); \ + } while (false) +#endif + +#ifdef NDEBUG +#define ifdebug if (false) +#else +#define ifdebug if (true) +#endif + +#define __FILENAME__ (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__) + +#define LogPrint(flag, format, ...) \ + do { \ + if (hsa_flag_isset64(log_flags, flag)) \ + rocr::log_printf(__FILENAME__, __LINE__, format, ##__VA_ARGS__); \ + } while (false); + + +// A macro to disallow the copy and move constructor and operator= functions +#define DISALLOW_COPY_AND_ASSIGN(TypeName) \ + TypeName(const TypeName&) = delete; \ + TypeName(TypeName&&) = delete; \ + void operator=(const TypeName&) = delete; \ + void operator=(TypeName&&) = delete; + +template +class ScopeGuard { + public: + explicit __forceinline ScopeGuard(const lambda& release) + : release_(release), dismiss_(false) {} + + ScopeGuard(ScopeGuard& rhs) { *this = rhs; } + + __forceinline ~ScopeGuard() { + if (!dismiss_) release_(); + } + __forceinline ScopeGuard& operator=(ScopeGuard& rhs) { + dismiss_ = rhs.dismiss_; + release_ = rhs.release_; + rhs.dismiss_ = true; + return *this; + } + __forceinline void Dismiss() { dismiss_ = true; } + + private: + lambda release_; + bool dismiss_; +}; + +template +static __forceinline ScopeGuard MakeScopeGuard(lambda rel) { + return ScopeGuard(rel); +} + +#define MAKE_SCOPE_GUARD_HELPER(lname, sname, ...) \ + auto lname = __VA_ARGS__; \ + ScopeGuard sname(lname); +#define MAKE_SCOPE_GUARD(...) \ + MAKE_SCOPE_GUARD_HELPER(PASTE(scopeGuardLambda, __COUNTER__), \ + PASTE(scopeGuard, __COUNTER__), __VA_ARGS__) +#define MAKE_NAMED_SCOPE_GUARD(name, ...) \ + MAKE_SCOPE_GUARD_HELPER(PASTE(scopeGuardLambda, __COUNTER__), name, \ + __VA_ARGS__) + +/// @brief: Finds out the min one of two inputs, input must support ">" +/// operator. +/// @param: a(Input), a reference to type T. +/// @param: b(Input), a reference to type T. +/// @return: T. +template +static __forceinline T Min(const T& a, const T& b) { + return (a > b) ? b : a; +} + +template +static __forceinline T Min(const T& a, const T& b, Arg... args) { + return Min(a, Min(b, args...)); +} + +/// @brief: Find out the max one of two inputs, input must support ">" operator. +/// @param: a(Input), a reference to type T. +/// @param: b(Input), a reference to type T. +/// @return: T. +template +static __forceinline T Max(const T& a, const T& b) { + return (b > a) ? b : a; +} + +template +static __forceinline T Max(const T& a, const T& b, Arg... args) { + return Max(a, Max(b, args...)); +} + +/// @brief: Free the memory space which is newed previously. +/// @param: ptr(Input), a pointer to memory space. Can't be NULL. +/// @return: void. +struct DeleteObject { + template + void operator()(const T* ptr) const { + delete ptr; + } +}; + +/// @brief: Checks if a value is power of two, if it is, return true. Be careful +/// when passing 0. +/// @param: val(Input), the data to be checked. +/// @return: bool. +template +static __forceinline bool IsPowerOfTwo(T val) { + return (val & (val - 1)) == 0; +} + +/// @brief: Calculates the floor value aligned based on parameter of alignment. +/// If value is at the boundary of alignment, it is unchanged. +/// @param: value(Input), value to be calculated. +/// @param: alignment(Input), alignment value. +/// @return: T. +template +static __forceinline T AlignDown(T value, size_t alignment) { + return (T)((value / alignment) * alignment); +} + +/// @brief: Same as previous one, but first parameter becomes pointer, for more +/// info, see the previous desciption. +/// @param: value(Input), pointer to type T. +/// @param: alignment(Input), alignment value. +/// @return: T*, pointer to type T. +template +static __forceinline T* AlignDown(T* value, size_t alignment) { + return (T*)AlignDown((intptr_t)value, alignment); +} + +/// @brief: Calculates the ceiling value aligned based on parameter of +/// alignment. +/// If value is at the boundary of alignment, it is unchanged. +/// @param: value(Input), value to be calculated. +/// @param: alignment(Input), alignment value. +/// @param: T. +template +static __forceinline T AlignUp(T value, size_t alignment) { + return AlignDown((T)(value + alignment - 1), alignment); +} + +/// @brief: Same as previous one, but first parameter becomes pointer, for more +/// info, see the previous desciption. +/// @param: value(Input), pointer to type T. +/// @param: alignment(Input), alignment value. +/// @return: T*, pointer to type T. +template +static __forceinline T* AlignUp(T* value, size_t alignment) { + return (T*)AlignDown((intptr_t)((uint8_t*)value + alignment - 1), alignment); +} + +/// @brief: Checks if the input value is at the boundary of alignment, if it is, +/// @return true. +/// @param: value(Input), value to be checked. +/// @param: alignment(Input), alignment value. +/// @return: bool. +template +static __forceinline bool IsMultipleOf(T value, size_t alignment) { + return (AlignUp(value, alignment) == value); +} + +/// @brief: Same as previous one, but first parameter becomes pointer, for more +/// info, see the previous desciption. +/// @param: value(Input), pointer to type T. +/// @param: alignment(Input), alignment value. +/// @return: bool. +template +static __forceinline bool IsMultipleOf(T* value, size_t alignment) { + return (AlignUp(value, alignment) == value); +} + +static __forceinline uint32_t NextPow2(uint32_t value) { + if (value == 0) return 1; + uint32_t v = value - 1; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + return v + 1; +} + +static __forceinline uint64_t NextPow2(uint64_t value) { + if (value == 0) return 1; + uint64_t v = value - 1; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + v |= v >> 32; + return v + 1; +} + +static __forceinline bool strIsEmpty(const char* str) noexcept { return str[0] == '\0'; } + +static __forceinline std::string& ltrim(std::string& s) { + auto it = std::find_if(s.begin(), s.end(), + [](char c) { return !std::isspace(c, std::locale::classic()); }); + s.erase(s.begin(), it); + return s; +} + +static __forceinline std::string& rtrim(std::string& s) { + auto it = std::find_if(s.rbegin(), s.rend(), + [](char c) { return !std::isspace(c, std::locale::classic()); }); + s.erase(it.base(), s.end()); + return s; +} + +static __forceinline std::string& trim(std::string& s) { return ltrim(rtrim(s)); } + +} // namespace rocr + +template +static __forceinline uint32_t BitSelect(T p) { + static_assert(sizeof(T) <= sizeof(uintptr_t), "Type out of range."); + static_assert(highBit < sizeof(uintptr_t) * 8, "Bit index out of range."); + + uintptr_t ptr = p; + if (highBit != (sizeof(uintptr_t) * 8 - 1)) + return (uint32_t)((ptr & ((1ull << (highBit + 1)) - 1)) >> lowBit); + else + return (uint32_t)(ptr >> lowBit); +} + +inline uint32_t PtrLow16Shift8(const void* p) { + uintptr_t ptr = reinterpret_cast(p); + return (uint32_t)((ptr & 0xFFFFULL) >> 8); +} + +inline uint32_t PtrHigh64Shift16(const void* p) { + uintptr_t ptr = reinterpret_cast(p); + return (uint32_t)((ptr & 0xFFFFFFFFFFFF0000ULL) >> 16); +} + +inline uint32_t PtrLow40Shift8(const void* p) { + uintptr_t ptr = reinterpret_cast(p); + return (uint32_t)((ptr & 0xFFFFFFFFFFULL) >> 8); +} + +inline uint32_t PtrHigh64Shift40(const void* p) { + uintptr_t ptr = reinterpret_cast(p); + return (uint32_t)((ptr & 0xFFFFFF0000000000ULL) >> 40); +} + +static inline uint8_t Ptr48High8(const void* p) { + uintptr_t ptr = reinterpret_cast(p); + return (uint8_t)((ptr & 0xFF0000000000ULL) >> 40); +} + +static inline uint32_t Ptr48Low32(const void* p) { + uintptr_t ptr = reinterpret_cast(p); + assert((ptr & 0xFFFFFFFFFF00ULL) == ptr); + return (uint32_t)((ptr & 0xFFFFFFFFFFULL) >> 8); +} + +inline uint32_t PtrLow32(const void* p) { + return static_cast(reinterpret_cast(p)); +} + +inline uint32_t PtrHigh32(const void* p) { + uint32_t ptr = 0; +#ifdef HSA_LARGE_MODEL + ptr = static_cast(reinterpret_cast(p) >> 32); +#endif + return ptr; +} + +inline uint32_t HighPart(uint64_t value) { + return (value & 0xFFFFFFFF00000000) >> 32; +} + +inline uint32_t LowPart(uint64_t value) { + return (value & 0x00000000FFFFFFFF); +} + +#include "atomic_helpers.h" + +#endif // HSA_RUNTIME_CORE_UTIL_UTILS_H_ diff --git a/util/win/os_win.cpp b/util/win/os_win.cpp new file mode 100644 index 0000000000..81c90cd266 --- /dev/null +++ b/util/win/os_win.cpp @@ -0,0 +1,327 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +#ifdef _WIN32 // Are we compiling for windows? +#define NOMINMAX + +#include "core/util/os.h" + +#include +#include +#include +#include + +#include +#include +#include + +#undef Yield +#undef CreateMutex + +namespace rocr { +namespace os { + +static_assert(sizeof(LibHandle) == sizeof(HMODULE), + "OS abstraction size mismatch"); +static_assert(sizeof(LibHandle) == sizeof(::HANDLE), + "OS abstraction size mismatch"); +static_assert(sizeof(Semaphore) == sizeof(::HANDLE), + "OS abstraction size mismatch"); +static_assert(sizeof(Mutex) == sizeof(::HANDLE), + "OS abstraction size mismatch"); +static_assert(sizeof(Thread) == sizeof(::HANDLE), + "OS abstraction size mismatch"); +static_assert(sizeof(EventHandle) == sizeof(::HANDLE), + "OS abstraction size mismatch"); + +LibHandle LoadLib(std::string filename) { + HMODULE ret = LoadLibrary(filename.c_str()); + return *(LibHandle*)&ret; +} + +void* GetExportAddress(LibHandle lib, std::string export_name) { + return GetProcAddress(*(HMODULE*)&lib, export_name.c_str()); +} + +void CloseLib(LibHandle lib) { FreeLibrary(*(::HMODULE*)&lib); } + +std::vector GetLoadedLibs() { + // Use EnumProcessModulesEx + static_assert(false, "Not implemented."); +} + +std::string GetLibraryName(LibHandle lib) { + static_assert(false, "Not implemented."); +} + +Semaphore CreateSemaphore() { + sem = static_cast(CreateSemaphore(NULL, 0, LONG_MAX, NULL)); + assert(sem != NULL && "CreateSemaphore failed"); + + return *(Semaphore*)&sem; +} + +bool WaitSemaphore(Semaphore sem) { + return WaitForSingleObject(*(::HANDLE*)&lock, INFINITE) == WAIT_OBJECT_0; +} + +void PostSemaphore(Semaphore sem) { + ReleaseSemaphore(static_cast(*sem), 1, NULL); +} + +void DestroySemaphore(Semaphore sem) { + if (!CloseHandle(static_cast(*sem))) { + assert("CloseHandle() failed"); + } + *sem = NULL; +} + +Mutex CreateMutex() { return CreateEvent(NULL, false, true, NULL); } + +bool TryAcquireMutex(Mutex lock) { + return WaitForSingleObject(*(::HANDLE*)&lock, 0) == WAIT_OBJECT_0; +} + +bool AcquireMutex(Mutex lock) { + return WaitForSingleObject(*(::HANDLE*)&lock, INFINITE) == WAIT_OBJECT_0; +} + +void ReleaseMutex(Mutex lock) { SetEvent(*(::HANDLE*)&lock); } + +void DestroyMutex(Mutex lock) { CloseHandle(*(::HANDLE*)&lock); } + +void Sleep(int delay_in_millisecond) { ::Sleep(delay_in_millisecond); } + +void uSleep(int delayInUs) { ::Sleep(delayInUs / 1000); } + +void YieldThread() { ::Sleep(0); } + +struct ThreadArgs { + void* entry_args; + ThreadEntry entry_function; +}; + +unsigned __stdcall ThreadTrampoline(void* arg) { + ThreadArgs* thread_args = (ThreadArgs*)arg; + ThreadEntry entry = thread_args->entry_function; + void* data = thread_args->entry_args; + delete thread_args; + entry(data); + _endthreadex(0); + return 0; +} + +Thread CreateThread(ThreadEntry entry_function, void* entry_argument, + uint stack_size) { + ThreadArgs* thread_args = new ThreadArgs(); + thread_args->entry_args = entry_argument; + thread_args->entry_function = entry_function; + uintptr_t ret = + _beginthreadex(NULL, stack_size, ThreadTrampoline, thread_args, 0, NULL); + return *(Thread*)&ret; +} + +void CloseThread(Thread thread) { CloseHandle(*(::HANDLE*)&thread); } + +bool WaitForThread(Thread thread) { + return WaitForSingleObject(*(::HANDLE*)&thread, INFINITE) == WAIT_OBJECT_0; +} + +bool WaitForAllThreads(Thread* threads, uint thread_count) { + return WaitForMultipleObjects(thread_count, threads, TRUE, INFINITE) == + WAIT_OBJECT_0; +} + +void SetEnvVar(std::string env_var_name, std::string env_var_value) { + SetEnvironmentVariable(env_var_name.c_str(), env_var_value.c_str()); +} + +std::string GetEnvVar(std::string env_var_name) { + char* buff; + DWORD char_count = GetEnvironmentVariable(env_var_name.c_str(), NULL, 0); + if (char_count == 0) return ""; + buff = (char*)alloca(sizeof(char) * char_count); + GetEnvironmentVariable(env_var_name.c_str(), buff, char_count); + buff[char_count - 1] = '\0'; + std::string ret = buff; + return ret; +} + +size_t GetUserModeVirtualMemorySize() { + SYSTEM_INFO system_info = {0}; + GetSystemInfo(&system_info); + return ((size_t)system_info.lpMaximumApplicationAddress + 1); +} + +size_t GetUsablePhysicalHostMemorySize() { + MEMORYSTATUSEX memory_status = {0}; + memory_status.dwLength = sizeof(memory_status); + if (GlobalMemoryStatusEx(&memory_status) == 0) { + return 0; + } + + const size_t physical_size = static_cast(memory_status.ullTotalPhys); + return std::min(GetUserModeVirtualMemorySize(), physical_size); +} + +uintptr_t GetUserModeVirtualMemoryBase() { return (uintptr_t)0; } + +// Os event wrappers +EventHandle CreateOsEvent(bool auto_reset, bool init_state) { + EventHandle evt = reinterpret_cast( + CreateEvent(NULL, (BOOL)(!auto_reset), (BOOL)init_state, NULL)); + return evt; +} + +int DestroyOsEvent(EventHandle event) { + if (event == NULL) { + return -1; + } + return CloseHandle(reinterpret_cast<::HANDLE>(event)); +} + +int WaitForOsEvent(EventHandle event, unsigned int milli_seconds) { + if (event == NULL) { + return -1; + } + + int ret_code = + WaitForSingleObject(reinterpret_cast<::HANDLE>(event), milli_seconds); + if (ret_code == WAIT_TIMEOUT) { + ret_code = 0x14003; // 0x14003 indicates timeout + } + return ret_code; +} + +int SetOsEvent(EventHandle event) { + if (event == NULL) { + return -1; + } + return SetEvent(reinterpret_cast<::HANDLE>(event)); +} + +int ResetOsEvent(EventHandle event) { + if (event == NULL) { + return -1; + } + return ResetEvent(reinterpret_cast<::HANDLE>(event)); +} + +uint64_t ReadAccurateClock() { + uint64_t ret; + QueryPerformanceCounter((LARGE_INTEGER*)&ret); + return ret; +} + +uint64_t AccurateClockFrequency() { + uint64_t ret; + QueryPerformanceFrequency((LARGE_INTEGER*)&ret); + return ret; +} + +SharedMutex CreateSharedMutex() { + assert(false && "Not implemented."); + abort(); + return nullptr; +} + +bool TryAcquireSharedMutex(SharedMutex lock) { + assert(false && "Not implemented."); + abort(); + return false; +} + +bool AcquireSharedMutex(SharedMutex lock) { + assert(false && "Not implemented."); + abort(); + return false; +} + +void ReleaseSharedMutex(SharedMutex lock) { + assert(false && "Not implemented."); + abort(); +} + +bool TrySharedAcquireSharedMutex(SharedMutex lock) { + assert(false && "Not implemented."); + abort(); + return false; +} + +bool SharedAcquireSharedMutex(SharedMutex lock) { + assert(false && "Not implemented."); + abort(); + return false; +} + +void SharedReleaseSharedMutex(SharedMutex lock) { + assert(false && "Not implemented."); + abort(); +} + +void DestroySharedMutex(SharedMutex lock) { + assert(false && "Not implemented."); + abort(); +} + +uint64_t ReadSystemClock() { + assert(false && "Not implemented."); + abort(); + return 0; +} + +uint64_t SystemClockFrequency() { + assert(false && "Not implemented."); + abort(); + return 0; +} + +bool ParseCpuID(cpuid_t* cpuinfo) { + assert(false && "Not implemented."); + abort(); + return false; +} + +} // namespace os +} // namespace rocr + +#endif diff --git a/version.cpp b/version.cpp new file mode 100644 index 0000000000..b71d036fc4 --- /dev/null +++ b/version.cpp @@ -0,0 +1,52 @@ +/* + * Copyright © 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "libhsakmt.h" + +HSAKMT_STATUS HSAKMTAPI hsaKmtGetVersion(HsaVersionInfo *VersionInfo) { + CHECK_DXG_OPEN(); + + VersionInfo->KernelInterfaceMajorVersion = 1; + VersionInfo->KernelInterfaceMinorVersion = 16; + + return HSAKMT_STATUS_SUCCESS; +} +HSAKMT_STATUS HSAKMTAPI +hsaKmtGetVersionCapInfo(HsaVersionCapability *VersionCapInfo) { + CHECK_DXG_OPEN(); + + VersionCapInfo->Value = 0; + /*VersionCapInfo->ui64.InterruptSignal = 0; + VersionCapInfo->ui64.Sdma = 0; + VersionCapInfo->ui64.SdmaXgmi = 0; + VersionCapInfo->ui64.Image = 0; + VersionCapInfo->ui64.EventAge = 0; + VersionCapInfo->ui64.Scratch = 0; + VersionCapInfo->ui64.Vmem = 0; + VersionCapInfo->ui64.dmabuf = 0; + VersionCapInfo->ui64.XNack = 0;*/ + + return HSAKMT_STATUS_SUCCESS; +} \ No newline at end of file diff --git a/wddm/cmd_util.cpp b/wddm/cmd_util.cpp new file mode 100644 index 0000000000..20e0231977 --- /dev/null +++ b/wddm/cmd_util.cpp @@ -0,0 +1,281 @@ +/* Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. */ + +#include "inc/wddm/cmd_util.h" + +namespace rocr { +namespace core { + +/* + * Builds a COPY_DATA packet that copies data. + */ +size_t CmdUtil::BuildCopyData( + uint64_t *pDstAddr, + void *pBuffer, + uint32_t dstSel, + uint32_t dstCachePolicy, + uint32_t srcSel, + uint32_t srcCachePolicy, + uint32_t countSel, + uint32_t wrConfirm) { + PM4MEC_COPY_DATA copy_data = {0}; + + GenerateCmdHeader(©_data, IT_COPY_DATA); + copy_data.bitfields2.dst_sel = dstSel; + copy_data.bitfields2.src_sel = srcSel; + copy_data.bitfields2.dst_cache_policy = dstCachePolicy; + copy_data.bitfields2.src_cache_policy = srcCachePolicy; + copy_data.bitfields2.count_sel = countSel; + copy_data.bitfields2.wr_confirm = wrConfirm; + copy_data.bitfields5c.dst_64b_addr_lo = (PtrLow32(pDstAddr) >> 3); + copy_data.dst_addr_hi = PtrHigh32(pDstAddr); + memcpy(pBuffer, ©_data, sizeof(copy_data)); + + return sizeof(copy_data); +} + +/* + * Builds a EVENT_WRITE packet. + * Applications can use Barrier command to ensure their + * command is executed only after all other commands have + * completed their execution. + */ +size_t CmdUtil::BuildBarrier( + void *pBuffer, + uint32_t eventIndex, + uint32_t eventType) { + BarrierTemplate barrier = {0}; + + GenerateCmdHeader(&barrier.event_write, IT_EVENT_WRITE); + barrier.event_write.bitfields2.event_index = eventIndex; + barrier.event_write.bitfields2.event_type = eventType; + memcpy(pBuffer, &barrier, sizeof(barrier)); + + return sizeof(barrier); +} + +/* + * Builds a ACQUIRE_MEM packet. + * Users can submit this command to + * invalidate Gpu caches - L1 and or L2. + */ +size_t CmdUtil::BuildAcquireMem( + uint8_t major, + void *pBuffer) { + size_t ret; + if (major == 9) { + gfx9::AcquireMemTemplate acq = {0}; + GenerateCmdHeader(&acq.acquire_mem, IT_ACQUIRE_MEM); + // Specify the size of memory to invalidate. Size is + // specified in terms of 256 byte chunks. A coher_size + // of 0xFFFFFFFF actually specified 0xFFFFFFFF00 (40 bits) + // of memory. The field coher_size_hi specifies memory from + // bits 40-64 for a total of 256 TB. + acq.acquire_mem.coher_size = 0xFFFFFFFF; + acq.acquire_mem.bitfields4.coher_size_hi = 0xFF; + // Specify the address of memory to invalidate. The + // address must be 256 byte aligned. + acq.acquire_mem.coher_base_lo = 0; + acq.acquire_mem.bitfields6.coher_base_hi = 0; + // Specify the poll interval for determing if operation is complete + acq.acquire_mem.bitfields7.poll_interval = 4; + acq.acquire_mem.bitfields2.coher_cntl = + (1 << 29) | // CP_COHER_CNTL__SH_ICACHE_ACTION_ENA_MASK + (1 << 27) | // CP_COHER_CNTL__SH_KCACHE_ACTION_ENA_MASK + (1 << 28); // CP_COHER_CNTL__SH_KCACHE_VOL_ACTION_ENA_MASK + memcpy(pBuffer, &acq, sizeof(acq)); + ret = sizeof(acq); + } else if (major >= 10) { + gfx10::AcquireMemTemplate acq = {0}; + GenerateCmdHeader(&acq.acquire_mem, IT_ACQUIRE_MEM); + acq.acquire_mem.coher_size = 0xFFFFFFFF; + acq.acquire_mem.bitfields4.coher_size_hi = 0xFF; + acq.acquire_mem.coher_base_lo = 0; + acq.acquire_mem.bitfields6.coher_base_hi = 0; + acq.acquire_mem.bitfields7.poll_interval = 4; + acq.acquire_mem.bitfields8.gcr_cntl = + (1 << 16) | // SEQ = FORWARD + (1 << 15) | // GL2_WB + (1 << 14) | // GL2_INV + (1 << 9) | // GL1_INV + (1 << 8) | // GLV_INV + (1 << 7) | // GLK_INV + (1 << 6) | // GLK_WB + (1 << 5) | // GLM_INV + (1 << 4) | // GLM_WB + (1 << 0); // GLI_INV = ALL + memcpy(pBuffer, &acq, sizeof(acq)); + ret = sizeof(acq); + } + + return ret; +} + +/* + * Builds a scratch packet. + */ +size_t CmdUtil::BuildScratch( + void *pScratchBase, + void *pBuffer) { + struct SetScratchTemplate scratch = {0}; + + GenerateSetShRegHeader(&scratch, mmCOMPUTE_DISPATCH_SCRATCH_BASE_LO); + scratch.scratch_lo = Ptr48Low32(pScratchBase); + scratch.scratch_hi = Ptr48High8(pScratchBase); + memcpy(pBuffer, &scratch, sizeof(scratch)); + + return sizeof(scratch); +} + +/** + * @ Set Compute Shader parameter for gfx11 and above + */ +size_t CmdUtil::BuildComputeShaderParams(void *pBuffer) { + struct DispatchProgramResourceRegs compute_shader_params = {0}; + + GenerateSetShRegHeader(&compute_shader_params, mmCOMPUTE_PGM_RSRC3); + // IMAGE_OP: Indicates the compute program contains an image op + // instruction and should be stalled by its WAIT_SYNC fence. + compute_shader_params.compute_pgm_rsrc3 = (1 << 31); + + memcpy(pBuffer, &compute_shader_params, sizeof(compute_shader_params)); + + return sizeof(compute_shader_params); +} + + +/* + * Builds a dispatch packet. + */ +size_t CmdUtil::BuildDispatch( + struct DispatchInfo *pInfo, + void *pBuffer) { + DispatchTemplate dispatch = {0}; + + GenerateSetShRegHeader(&dispatch.dimension_regs, mmCOMPUTE_NUM_THREAD_X); + dispatch.dimension_regs.compute_num_thread_x = pInfo->pPacket->workgroup_size_x; + dispatch.dimension_regs.compute_num_thread_y = pInfo->pPacket->workgroup_size_y; + dispatch.dimension_regs.compute_num_thread_z = pInfo->pPacket->workgroup_size_z; + + // TODO: Add AQL packet index for debugger + // Debugger requires AQL packet index in COMPUTE_DISPATCH_PKT_ADDR_LO + GenerateSetShRegHeader(&dispatch.program_regs, mmCOMPUTE_PGM_LO); + dispatch.program_regs.compute_pgm_lo = Ptr48Low32(pInfo->pEntry); + dispatch.program_regs.compute_pgm_hi = Ptr48High8(pInfo->pEntry); + + GenerateSetShRegHeader(&dispatch.program_resource_regs, mmCOMPUTE_PGM_RSRC1); + dispatch.program_resource_regs.compute_pgm_rsrc1 = pInfo->pKernelObject->compute_pgm_rsrc1; + dispatch.program_resource_regs.compute_pgm_rsrc2 = + (pInfo->ldsBlks << 15) | pInfo->pKernelObject->compute_pgm_rsrc2; + + GenerateSetShRegHeader(&dispatch.resource_regs, mmCOMPUTE_RESOURCE_LIMITS); + dispatch.resource_regs.compute_resource_limits = 0x3ff; + dispatch.resource_regs.compute_static_thread_mgmt_se0 = 0xFFFFFFFF; + dispatch.resource_regs.compute_static_thread_mgmt_se1 = 0xFFFFFFFF; + dispatch.resource_regs.compute_static_thread_mgmt_se2 = 0xFFFFFFFF; + dispatch.resource_regs.compute_static_thread_mgmt_se3 = 0xFFFFFFFF; + + dispatch.resource_regs.compute_tmpring_size = pInfo->pAmdQueue->compute_tmpring_size; + + GenerateSetShRegHeader(&dispatch.compute_user_data_regs, mmCOMPUTE_USER_DATA_0); + + uint32_t sgpr_no = 0; + if (AMD_HSA_BITS_GET(pInfo->pKernelObject->kernel_code_properties, + AMD_KERNEL_CODE_PROPERTIES_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER)) { + assert(pInfo->major < 11); + pInfo->scratchBaseOffset[pInfo->offsetCnt++] = + offsetof(struct DispatchTemplate, compute_user_data_regs.compute_user_data[0]) + + sgpr_no * sizeof(uint32_t); + + dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] = + pInfo->pAmdQueue->scratch_resource_descriptor[0]; + dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] = + pInfo->pAmdQueue->scratch_resource_descriptor[1]; + dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] = + pInfo->pAmdQueue->scratch_resource_descriptor[2]; + dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] = + pInfo->srd; + } + if (AMD_HSA_BITS_GET(pInfo->pKernelObject->kernel_code_properties, + AMD_KERNEL_CODE_PROPERTIES_ENABLE_SGPR_DISPATCH_PTR)) { + dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] = PtrLow32(pInfo->pPacket); + dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] = PtrHigh32(pInfo->pPacket); + } + if (AMD_HSA_BITS_GET(pInfo->pKernelObject->kernel_code_properties, + AMD_KERNEL_CODE_PROPERTIES_ENABLE_SGPR_QUEUE_PTR)) { + dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] = PtrLow32(pInfo->pAmdQueue); + dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] = PtrHigh32(pInfo->pAmdQueue); + } + if (AMD_HSA_BITS_GET(pInfo->pKernelObject->kernel_code_properties, + AMD_KERNEL_CODE_PROPERTIES_ENABLE_SGPR_KERNARG_SEGMENT_PTR)) { + dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] = + PtrLow32(pInfo->pPacket->kernarg_address); + dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] = + PtrHigh32(pInfo->pPacket->kernarg_address); + } + if (AMD_HSA_BITS_GET(pInfo->pKernelObject->kernel_code_properties, + AMD_KERNEL_CODE_PROPERTIES_ENABLE_SGPR_DISPATCH_ID)) { + // This feature may be enabled as a side effect of indirect calls. + // However, the compiler team confirmed that the dispatch id itself is not used, + // so safe to send 0 for each dispatch. + dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] = NULL; + dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] = NULL; + } + if (AMD_HSA_BITS_GET(pInfo->pKernelObject->kernel_code_properties, + AMD_KERNEL_CODE_PROPERTIES_ENABLE_SGPR_FLAT_SCRATCH_INIT)) { + assert(pInfo->major < 11); + pInfo->scratchBaseOffset[pInfo->offsetCnt++] = + offsetof(struct DispatchTemplate, compute_user_data_regs.compute_user_data[0]) + + sgpr_no * sizeof(uint32_t); + + dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] = + PtrLow32(pInfo->pScratchBase); + dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] = + PtrHigh32(pInfo->pScratchBase); + } + if (AMD_HSA_BITS_GET(pInfo->pKernelObject->kernel_code_properties, + AMD_KERNEL_CODE_PROPERTIES_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE)) { + dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] = + pInfo->scratchSizePerWave / (pInfo->wave32 ? 32 : 64); + } + + GenerateCmdHeader(&dispatch.dispatch_direct, IT_DISPATCH_DIRECT); + dispatch.dispatch_direct.dispatch_initiator = + (1 << 0) | // COMPUTE_SHADER_EN + (1 << 2) | // FORCE_START_AT_000 + (1 << 5); // USE_THREAD_DIMENSIONS + if (pInfo->wave32) dispatch.dispatch_direct.dispatch_initiator |= (1 << 15); // CS_W32_EN + dispatch.dispatch_direct.dim_x = pInfo->pPacket->grid_size_x; + dispatch.dispatch_direct.dim_y = pInfo->pPacket->grid_size_y; + dispatch.dispatch_direct.dim_z = pInfo->pPacket->grid_size_z; + memcpy(pBuffer, &dispatch, sizeof(dispatch)); + + return sizeof(dispatch); +} + +/* + * Builds a ATOMIC_MEM packet. + * Users can submit this command + * to perform atomic operations. + */ +size_t CmdUtil::BuildAtomicMem( + uint64_t *pAddr, + uint32_t atomic, + void *pBuffer, + uint32_t cachePolicy, + uint64_t srcData) { + AtomicTemplate atom = {0}; + + GenerateCmdHeader(&atom.atomic, IT_ATOMIC_MEM); + atom.atomic.addr_lo = PtrLow32(pAddr); + atom.atomic.addr_hi = PtrHigh32(pAddr); + atom.atomic.bitfields2.atomic = atomic; + atom.atomic.bitfields2.cache_policy = cachePolicy; + atom.atomic.src_data_lo = LowPart(srcData); + atom.atomic.src_data_hi = HighPart(srcData); + memcpy(pBuffer, &atom, sizeof(atom)); + + return sizeof(atom); +} + +} // namespace core +} // namespace rocr diff --git a/wddm/device.cpp b/wddm/device.cpp new file mode 100644 index 0000000000..753f100a92 --- /dev/null +++ b/wddm/device.cpp @@ -0,0 +1,879 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2020, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +#include +#include + +#include +#include +#include +#include +#include +#include "inc/wddm/status.h" +#include "inc/wddm/types.h" +#include "inc/wddm/device.h" +#include "inc/wddm/queue.h" + +namespace rocr { +namespace core { + +const uint32_t WDDMDevice::cmdbuf_aql_frame_num_ = 0x1000; + +WDDMDevice::WDDMDevice(D3DKMT_HANDLE adapter, LUID adapter_luid) + : adapter_(adapter), adapter_luid_(adapter_luid) { + memset(&device_info_, 0, sizeof(device_info_)); + + ParseDeviceInfo(); + CreateDevice(); + SetPowerOptimization(false); + CreatePagingQueue(); + ReserveLocalHeapSpace(); + ReserveSystemHeapSpace(); + InitVaMgr(); + InitCmdbufInfo(); +} + +WDDMDevice::~WDDMDevice() { + FreeLocalHeapSpace(); + FreeSystemHeapSpace(); + DestroyPagingQueue(); + SetPowerOptimization(true); + DestroyDevice(); + + DestroyDeviceInfo(); +} + +static NTSTATUS WDDMQueryAdapter(D3DKMT_HANDLE adapter, KMTQUERYADAPTERINFOTYPE type, + void *data, int size) +{ + D3DKMT_QUERYADAPTERINFO args = {0}; + + args.hAdapter = adapter; + args.Type = type; + args.pPrivateDriverData = data; + args.PrivateDriverDataSize = size; + + return D3DKMTQueryAdapterInfo(&args); +} + +uint64_t WDDMDevice::VramAvail(void) { + D3DKMT_QUERYSTATISTICS stats; + NTSTATUS ret; + uint64_t usedVis = 0; + uint64_t usedInv = 0; + + // wait fence complete + uint64_t value = page_fence_value_.load(); + if(!CpuWait(&page_syncobj_, &value, 1, false)) + return HSA_STATUS_ERROR; + + // local cpu-visible memory + memset(&stats, 0, sizeof(D3DKMT_QUERYSTATISTICS)); + stats.Type = D3DKMT_QUERYSTATISTICS_SEGMENT; + stats.AdapterLuid = adapter_luid_; + stats.QuerySegment.SegmentId = 0; + ret = D3DKMTQueryStatistics(&stats); + if (ret == 0) + usedVis = stats.QueryResult.SegmentInformation.BytesResident; + + // local invisible memory + memset(&stats, 0, sizeof(D3DKMT_QUERYSTATISTICS)); + stats.Type = D3DKMT_QUERYSTATISTICS_SEGMENT; + stats.AdapterLuid = adapter_luid_; + stats.QuerySegment.SegmentId = 1; + + ret = D3DKMTQueryStatistics(&stats); + if (ret == 0) + usedInv = stats.QueryResult.SegmentInformation.BytesResident; + + return LocalHeapSize() - usedVis - usedInv; +} + +bool WDDMDevice::CreateDevice(void) { + D3DKMT_CREATEDEVICE args = {0}; + args.hAdapter = adapter_; + + NTSTATUS ret = D3DKMTCreateDevice(&args); + if (ret == STATUS_SUCCESS) { + device_ = args.hDevice; + return true; + } + + fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret); + return false; +} + +bool WDDMDevice::DestroyDevice(void) { + D3DKMT_DESTROYDEVICE args = {0}; + args.hDevice = device_; + + NTSTATUS ret = D3DKMTDestroyDevice(&args); + if (ret == STATUS_SUCCESS) + return true; + + fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret); + return false; +} + +bool WDDMDevice::CreatePagingQueue(void) { + D3DKMT_CREATEPAGINGQUEUE args = {0}; + args.hDevice = device_; + args.Priority = D3DDDI_PAGINGQUEUE_PRIORITY_NORMAL; + + NTSTATUS ret = D3DKMTCreatePagingQueue(&args); + if (ret == STATUS_SUCCESS) { + page_queue_ = args.hPagingQueue; + page_syncobj_ = args.hSyncObject; + page_fence_addr_ = (uint64_t *)args.FenceValueCPUVirtualAddress; + page_fence_value_ = 0; + return true; + } + + fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret); + return false; +} + +bool WDDMDevice::DestroyPagingQueue(void) { + D3DDDI_DESTROYPAGINGQUEUE args = {0}; + args.hPagingQueue = page_queue_; + + NTSTATUS ret = D3DKMTDestroyPagingQueue(&args); + if (ret == STATUS_SUCCESS) + return true; + + fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret); + return false; +} + +bool WDDMDevice::CommitSystemHeapSpace(void* addr, int64_t size, bool lock) { + int32_t protFlags = PROT_READ | PROT_WRITE | PROT_EXEC; + int32_t mapFlags = MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED| + MAP_NORESERVE|MAP_UNINITIALIZED; + if (lock) + mapFlags |= MAP_LOCKED; + void* paddr = mmap(addr, size, protFlags, mapFlags, -1, 0); + if (paddr == MAP_FAILED) { + fprintf(stderr, "%s fail to commit %s addr = %p, paddr = %p\n", + __FUNCTION__, (lock ? "locked" : ""), addr, paddr); + return false; + } + assert(addr == paddr); + + /*if (!Runtime::runtime_singleton_->PinWARequired()) + return true;*/ + + /* + * Do not make the pages in this range available to the child + * after a fork(2). This is useful to prevent copy-on-write + * semantics from changing the physical location of a page if + * the parent writes to it after a fork(2). (Such page + * relocations cause problems for hardware that DMAs into the + * page.) + * + * https://man7.org/linux/man-pages/man2/madvise.2.html + */ + if (madvise(addr, size, MADV_DONTFORK)) + fprintf(stderr, "%s fail to set MADV_DONTFORK for addr = %p\n", + __FUNCTION__, addr); + + return true; +} + +bool WDDMDevice::DecommitSystemHeapSpace(void* addr, int64_t size) { + int32_t protFlags = PROT_NONE; + int32_t mapFlags = MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED| + MAP_NORESERVE|MAP_UNINITIALIZED; + void* paddr = mmap(addr, size, protFlags, mapFlags, -1, 0); + if (paddr == MAP_FAILED) { + fprintf(stderr, "%s fail to decommit addr = %p, paddr = %p\n", + __FUNCTION__, addr, paddr); + return false; + } + assert(addr == paddr); + return true; +} + +bool WDDMDevice::ReserveSystemHeapSpace() { + struct sysinfo info; + int ret = sysinfo(&info); + uint64_t max_ram = 0x10000000000; + uint64_t alignment = 0x100000000; + assert(!ret); + + int32_t protFlags = PROT_NONE; + // minimum of reserve size is 8G, maximum of reserve size is 1T. + system_heap_space_size_ = std::min(AlignUp(info.totalram, alignment) * 2, max_ram); + void* cpu = mmap(NULL, system_heap_space_size_, protFlags, + MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + if (cpu == MAP_FAILED) { + fprintf(stderr, "%s fail to reserve system_heap_space_size_ = %lx \n", + __FUNCTION__, system_heap_space_size_); + return false; + } + + system_heap_space_start_ = (uint64_t)cpu; + return true; +} + +bool WDDMDevice::FreeSystemHeapSpace(void) { + void *cpu = (void *)system_heap_space_start_; + if (munmap(cpu, system_heap_space_size_ != 0)) { + fprintf(stderr, "%s fail to unmap = %p \n", __FUNCTION__, cpu); + return false; + } + return true; +} + +/* + * To find the avaliable same range for cpu + * virtual space and gpu virtual space. + * sys_va_size of cpu va range is larger 1G + * than gpu va range, otherwise ReserveGPUVirtualAddress + * will return error. + */ +bool WDDMDevice::ReserveLocalHeapSpace(void) { + uint64_t sys_va[16] = {0}; + uint64_t local_va; + uint64_t sys_va_size; + int match_index = -1; + uint64_t align = 0x40000000; /* 1G */ + void* ptr = NULL; + + local_heap_space_start_ = 0; + local_heap_space_size_ = AlignUp(LocalHeapSize(), align) * 4; + sys_va_size = local_heap_space_size_ + align; + + /* it will retry 16 times to find the avaliable range. */ + for (int i = 0; i < 16; i++) { + local_va = 0; + ptr = mmap(NULL, sys_va_size , PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + if (ptr == MAP_FAILED) { + fprintf(stderr, "%s fail to reserve cpu va in %d time!\n", + __FUNCTION__, i); + break; + } + + sys_va[i] = (uint64_t)ptr; + + if (thunk::ReserveGpuVirtualAddress( + adapter_, local_heap_space_size_, + (uint64_t)ptr, + (uint64_t)ptr + sys_va_size, &local_va) == ErrorCode::Success) { + + match_index = i; + local_heap_space_start_ = local_va; + debug_print("%s success to reserve gpu va %lx and va cpu %p in %d time\n", + __FUNCTION__, local_va, ptr, i); + break; + } else { + debug_print("%s fail to reserve gpu va for cpu va %p in %d time!\n", + __FUNCTION__, ptr, i); + } + } + + if (match_index >= 0) { + /* release cpu unused ranges*/ + uint64_t left_size = local_va - sys_va[match_index]; + uint64_t right_size = align - left_size; + if ((left_size > 0) && munmap((void*)sys_va[match_index], left_size)) + fprintf(stderr, "%s fail to unmap left %lx with size %lx\n", + __FUNCTION__, sys_va[match_index], left_size); + if ((right_size > 0) && munmap((void*)(local_va + local_heap_space_size_), right_size)) + fprintf(stderr, "%s fail to unmap right %lx with size %lx\n", + __FUNCTION__, (local_va + local_heap_space_size_), right_size); + } else { + fprintf(stderr, "%s fail to reserve Local Heap Space !\n", + __FUNCTION__); + } + + /* free match fail address for cpu va */ + int free = match_index >= 0 ? match_index : 16; + for (int j = 0; j < free; j++) { + if (sys_va[j] != 0 && munmap((void*)sys_va[j], sys_va_size)) { + fprintf(stderr, "%s fail to unmap %d %lx\n", __FUNCTION__, j, sys_va[j]); + } + } + + return match_index >= 0; +} + +bool WDDMDevice::FreeLocalHeapSpace(void) { + thunk::FreeGpuVirtualAddress(adapter_, local_heap_space_start_, local_heap_space_size_); + void *cpu = (void *)local_heap_space_start_; + return munmap(cpu, local_heap_space_size_) == 0; +} + +void WDDMDevice::InitVaMgr() { + uint32_t min_align = 4096; + local_va_mgr_ = std::make_unique(local_heap_space_start_, local_heap_space_size_, min_align); +} + +void WDDMDevice::SetPowerOptimization(bool restore) { + void *priv_data; + int priv_size; + + priv_size = rocr_proxy::CreatePowerOptPrivData(&priv_data, restore); + + D3DKMT_ESCAPE d3dkmt_escape; + memset(&d3dkmt_escape, 0, sizeof(d3dkmt_escape)); + + d3dkmt_escape.hAdapter = adapter_; + d3dkmt_escape.hDevice = device_; + d3dkmt_escape.hContext = 0; //KMD only use device to identify the process + d3dkmt_escape.Type = D3DKMT_ESCAPE_DRIVERPRIVATE; + d3dkmt_escape.pPrivateDriverData = priv_data; + d3dkmt_escape.PrivateDriverDataSize = priv_size; + d3dkmt_escape.Flags.HardwareAccess = true; + + NTSTATUS status = D3DKMTEscape(&d3dkmt_escape); + debug_print("%s status %d restore %d\n", __FUNCTION__, status, restore); + rocr_proxy::DestroyPrivData(priv_data); +} + +ErrorCode WDDMDevice::ReserveGpuVirtualAddress(const rocr_proxy::AllocDomain domain, + gpusize hit_base_addr, gpusize size, + gpusize *out_gpu_virt_addr, gpusize alignment, bool lock) { + gpusize gpu_addr = 0; + ErrorCode code = ErrorCode::Success; + + if (domain == rocr_proxy::kSystem) { + + code = thunk::ReserveGpuVirtualAddress(adapter_, size, + system_heap_space_start_, + system_heap_space_start_ + system_heap_space_size_, + &gpu_addr); + if (code != ErrorCode::Success) + return code; + + if (!CommitSystemHeapSpace((void*)gpu_addr, size, lock)) { + thunk::FreeGpuVirtualAddress(adapter_, gpu_addr, size); + code = ErrorCode::SyscallFail; + } + } else { + uint64_t align = alignment == 0 ? (64 * 1024) : alignment; // default 64K alignment + if (domain == rocr_proxy::kLocal && size >= GPU_HUGE_PAGE_SIZE) + align = GPU_HUGE_PAGE_SIZE; + + gpu_addr = local_va_mgr_->Alloc(size, align, hit_base_addr); + if (gpu_addr == 0) + code = ErrorCode::OutOfGpuMemory; + + } + + *out_gpu_virt_addr = (code == ErrorCode::Success) ? gpu_addr : 0; + return code; +} + +ErrorCode WDDMDevice::FreeGpuVirtualAddress(const rocr_proxy::AllocDomain domain, + gpusize gpu_addr, gpusize size) { + auto code = ErrorCode::Success; + + if (domain == rocr_proxy::kSystem) { + + DecommitSystemHeapSpace((void *)gpu_addr, size); + + thunk::FreeGpuVirtualAddressArgs free_args{}; + free_args.hAdapter = adapter_; + free_args.BaseAddress = gpu_addr; + free_args.Size = size; + + code = thunk::FreeGpuVirtualAddress(&free_args); + } else { + local_va_mgr_->Free(gpu_addr); + } + + return code; +} + +void WDDMDevice::UpdatePageFence(uint64_t fence_value) { + uint64_t current = page_fence_value_.load(); + + // atomically set fence value when target is bigger than current one + do { + if (current >= fence_value) + break; + } while (!page_fence_value_.compare_exchange_weak(current, fence_value)); +} + +ErrorCode WDDMDevice::CreateGpuMemory(const GpuMemoryCreateInfo &create_info, GpuMemory **gpu_mem) { + ErrorCode ret; + + *gpu_mem = nullptr; + auto mem = new GpuMemory(this); + if (create_info.dmabuf_fd > 0) + ret = mem->ImportPhysicalHandle(create_info.dmabuf_fd); + else + ret = mem->Init(create_info); + if (ret == ErrorCode::Success) + *gpu_mem = mem; + + return ret; +} + +void *WDDMDevice::Lock(D3DKMT_HANDLE handle) { + D3DKMT_LOCK2 args = {0}; + args.hDevice = device_; + args.hAllocation = handle; + + NTSTATUS ret = D3DKMTLock2(&args); + if (ret == STATUS_SUCCESS) + return args.pData; + + fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret); + return NULL; +} + +bool WDDMDevice::Unlock(D3DKMT_HANDLE handle) { + D3DKMT_UNLOCK2 args = {0}; + args.hDevice = device_; + args.hAllocation = handle; + + NTSTATUS ret = D3DKMTUnlock2(&args); + if (ret == STATUS_SUCCESS) + return true; + + fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret); + return false; +} + +bool WDDMDevice::CreateContext(int engine, D3DKMT_HANDLE *handle) { + void *priv_data; + int priv_size; + + int ordinal = EngineOrdinal(engine, &device_info_); + if (ordinal < 0) + return false; + + bool FwManagedGfxState = SupportStateShadowingByCpFw(); + priv_size = rocr_proxy::CreateContextPrivData(&priv_data, FwManagedGfxState); + + D3DKMT_CREATECONTEXTVIRTUAL args = {0}; + args.hDevice = device_; + args.EngineAffinity = 1 << 0; + args.NodeOrdinal = ordinal; + args.pPrivateDriverData = priv_data; + args.PrivateDriverDataSize = priv_size; + args.ClientHint = D3DKMT_CLIENTHINT_OPENCL; + + if (IsHwsEnabled(engine)) + args.Flags.HwQueueSupported = 1; + else + args.Flags.DisableGpuTimeout = rocr_proxy::ShouldDisableGpuTimeout(engine, &device_info_); + + NTSTATUS ret = D3DKMTCreateContextVirtual(&args); + if (ret == STATUS_SUCCESS) { + *handle = args.hContext; + rocr_proxy::DestroyPrivData(priv_data); + return true; + } + + rocr_proxy::DestroyPrivData(priv_data); + + fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret); + return false; +} + +bool WDDMDevice::DestroyContext(D3DKMT_HANDLE handle) { + D3DKMT_DESTROYCONTEXT args = {0}; + args.hContext = handle; + + NTSTATUS ret = D3DKMTDestroyContext(&args); + if (ret == STATUS_SUCCESS) + return true; + + fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret); + return false; +} + +bool WDDMDevice::GpuWait(WDDMQueue *queue, const D3DKMT_HANDLE *syncobjs, + uint64_t *values, int count) { + + D3DKMT_WAITFORSYNCHRONIZATIONOBJECTFROMGPU args = {0}; + args.hContext = queue->context; + args.ObjectCount = count; + args.ObjectHandleArray = syncobjs; + args.MonitoredFenceValueArray = values; + + NTSTATUS ret = D3DKMTWaitForSynchronizationObjectFromGpu(&args); + if (ret == STATUS_SUCCESS) + return true; + + fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret); + return false; +} + +bool WDDMDevice::GpuSignal(D3DKMT_HANDLE context, const D3DKMT_HANDLE *syncobjs, + uint64_t *value, int count) { + D3DKMT_SIGNALSYNCHRONIZATIONOBJECTFROMGPU args = {0}; + args.hContext = context; + args.ObjectCount = count; + args.ObjectHandleArray = syncobjs; + args.MonitoredFenceValueArray = value; + + NTSTATUS ret = D3DKMTSignalSynchronizationObjectFromGpu(&args); + if (ret == STATUS_SUCCESS) + return true; + + fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret); + return false; +} + +bool WDDMDevice::CpuWait(const D3DKMT_HANDLE *syncobjs, uint64_t *value, + int count, bool wait_any) { + D3DKMT_WAITFORSYNCHRONIZATIONOBJECTFROMCPU args = {0}; + args.hDevice = device_; + args.ObjectCount = count; + args.ObjectHandleArray = syncobjs; + args.FenceValueArray = value; + args.Flags.WaitAny = wait_any; + + NTSTATUS ret = D3DKMTWaitForSynchronizationObjectFromCpu(&args); + if (ret == STATUS_SUCCESS) + return true; + + fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret); + return false; +} + +bool WDDMDevice::WaitOnPagingFenceFromCpu() { + uint64_t page_fence_value = 0; + + page_fence_value = page_fence_value_.load(); + if (CpuWait(&page_syncobj_, &page_fence_value, 1, false)) + return true; + + return false; +} + +bool WDDMDevice::CreateSyncobj(D3DKMT_HANDLE *handle, uint64_t **addr) { + D3DKMT_CREATESYNCHRONIZATIONOBJECT2 args = {0}; + args.hDevice = device_; + args.Info.Type = D3DDDI_MONITORED_FENCE; + args.Info.MonitoredFence.EngineAffinity = 1 << 0; + + NTSTATUS ret = D3DKMTCreateSynchronizationObject2(&args); + if (ret == STATUS_SUCCESS) { + *handle = args.hSyncObject; + *addr = (uint64_t *)args.Info.MonitoredFence.FenceValueCPUVirtualAddress; + debug_print("create syncobj cpu addr=%p gpu addr=%" PRIx64 "\n", + args.Info.MonitoredFence.FenceValueCPUVirtualAddress, + args.Info.MonitoredFence.FenceValueGPUVirtualAddress); + return true; + } + + fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret); + return false; +} + +void WDDMDevice::DestroySyncobj(D3DKMT_HANDLE handle) { + D3DKMT_DESTROYSYNCHRONIZATIONOBJECT args = {0}; + args.hSyncObject = handle; + + NTSTATUS ret = D3DKMTDestroySynchronizationObject(&args); + if (ret != STATUS_SUCCESS) + fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret); +} + +void WDDMDevice::InitCmdbufInfo(void) { + if (device_info_.major == 9) { + cmdbuf_aql_frame_size_ = 2 * sizeof(gfx9::AcquireMemTemplate); + } else if (device_info_.major >= 10) { + cmdbuf_aql_frame_size_ = 2 * sizeof(gfx10::AcquireMemTemplate); + } + + if (device_info_.major >= 11) + cmdbuf_aql_frame_size_ += sizeof(SetScratchTemplate); + + cmdbuf_aql_frame_size_ += + sizeof(PM4MEC_COPY_DATA) * 2 + + sizeof(BarrierTemplate) * 2 + + sizeof(DispatchTemplate) + + sizeof(AtomicTemplate) * 2; + cmdbuf_aql_frame_size_ = AlignUp(cmdbuf_aql_frame_size_, 0x10); + + cmdbuf_size_ = AlignUp(cmdbuf_aql_frame_num_ * cmdbuf_aql_frame_size_, 0x1000); +} + +uint32_t WDDMDevice::LdsBlocks(const hsa_kernel_dispatch_packet_t *pkt) { + static const uint32_t blk_sz = 512; + uint32_t total_sz = pkt->group_segment_size; + uint32_t blk_num = (total_sz + blk_sz - 1) / blk_sz; + return blk_num; +} + +NTSTATUS WDDMGetAdapters(D3DKMT_ADAPTERINFO *&adapters, int &num_adapters) +{ + bool supported = false; + D3DKMT_ENUMADAPTERS2 args = {0}; + NTSTATUS ret = D3DKMTEnumAdapters2(&args); + if (ret != STATUS_SUCCESS) + return ret; + + if (!args.NumAdapters) { + adapters = NULL; + num_adapters = 0; + return STATUS_SUCCESS; + } + + D3DKMT_ADAPTERINFO *info = new D3DKMT_ADAPTERINFO[args.NumAdapters]; + if (!info) + return STATUS_NO_MEMORY; + + args.pAdapters = info; + ret = D3DKMTEnumAdapters2(&args); + if (ret != STATUS_SUCCESS) + goto err_out0; + + adapters = new D3DKMT_ADAPTERINFO[args.NumAdapters]; + if (!adapters) + goto err_out0; + + num_adapters = 0; + for (int i = 0; i < args.NumAdapters; i++) { + D3DKMT_ADAPTERREGISTRYINFO query = {0}; + + ret = WDDMQueryAdapter(info[i].hAdapter, KMTQAITYPE_ADAPTERREGISTRYINFO, + &query, sizeof(query)); + if (ret != STATUS_SUCCESS) + goto err_out1; + + supported = rocr_proxy::QueryAdapterSupported(info[i].hAdapter); + + if (std::wcsstr(query.ChipType, L"AMD") && supported) { + adapters[num_adapters++] = info[i]; + } + } + + delete info; + return STATUS_SUCCESS; + + err_out1: + delete adapters; + adapters = NULL; + err_out0: + delete info; + return ret; +} + +bool WDDMDevice::ParseDeviceInfo() { + bool ret; + + memset(&device_info_, 0, sizeof(device_info_)); + ret = rocr_proxy::ParseAdapterInfo(adapter_, &device_info_); + if (!ret) + return false; + + return true; +} + +void WDDMDevice::DestroyDeviceInfo() { + free(device_info_.adapter_info); +} + +void WDDMDevice::GetClockCounters(uint64_t *gpu, uint64_t *cpu) { + void *priv_data; + int priv_size; + + priv_size = rocr_proxy::CreateCalibratedTimestampsPrivData(&priv_data); + + D3DKMT_ESCAPE d3dkmt_escape; + memset(&d3dkmt_escape, 0, sizeof(d3dkmt_escape)); + + d3dkmt_escape.hAdapter = adapter_; + d3dkmt_escape.hDevice = device_; + d3dkmt_escape.hContext = 0; //KMD only use device to identify the process + d3dkmt_escape.Type = D3DKMT_ESCAPE_DRIVERPRIVATE; + d3dkmt_escape.pPrivateDriverData = priv_data; + d3dkmt_escape.PrivateDriverDataSize = priv_size; + d3dkmt_escape.Flags.HardwareAccess = true; + + NTSTATUS status = D3DKMTEscape(&d3dkmt_escape); + if (status) { + debug_print("%s status %d \n", __FUNCTION__, status); + } else { + rocr_proxy::QueryCalibratedTimestamps(priv_data, gpu, cpu); + } + rocr_proxy::DestroyPrivData(priv_data); +} + +bool WDDMDevice::CreateQueue(WDDMQueue *queue) { + if (!CreateContext(queue->queue_engine, &queue->context)) + return false; + + GpuMemoryCreateInfo create_info{}; + create_info.size = queue->cmdbuf_size; + create_info.domain = rocr_proxy::kSystem; + + GpuMemory *gpu_mem = nullptr; + auto code = CreateGpuMemory(create_info, &gpu_mem); + if (code != ErrorCode::Success) + goto err_out0; + + queue->cmdbuf = gpu_mem->GetGpuMemoryHandle(); + queue->cmdbuf_addr = gpu_mem->GpuAddress(); + + if (queue->Init()) + goto err_out1; + + return true; + +err_out1: + delete gpu_mem; +err_out0: + DestroyContext(queue->context); + + return false; +} + +void WDDMDevice::DestroyQueue(WDDMQueue *queue) { + + queue->Fini(); + + auto cmdbuf_mem = GpuMemory::Convert(queue->cmdbuf); + delete cmdbuf_mem; + + DestroyContext(queue->context); +} + +bool WDDMDevice::SubmitToSwQueue(WDDMQueue *queue, uint64_t command_addr, + uint64_t command_size, uint64_t fence_value) { + void *priv_data; + int priv_size; + + priv_size = rocr_proxy::CreateSubmitPrivData(&priv_data, queue->queue, command_addr, command_size, false); + + D3DKMT_SUBMITCOMMAND args = {0}; + args.Commands = command_addr; + args.CommandLength = command_size; + args.BroadcastContextCount = 1; + args.BroadcastContext[0] = queue->context; + args.pPrivateDriverData = priv_data; + args.PrivateDriverDataSize = priv_size; + + NTSTATUS ret = D3DKMTSubmitCommand(&args); + if (ret != STATUS_SUCCESS) { + fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret); + rocr_proxy::DestroyPrivData(priv_data); + return false; + } + + rocr_proxy::DestroyPrivData(priv_data); + + if (!GpuSignal(queue->context, &queue->syncobj, &fence_value, 1)) + return false; + + return true; +} + +bool WDDMDevice::CreateHwQueue(WDDMQueue *queue) { + void *priv_data; + int priv_size; + + bool FwManagedGfxState = SupportStateShadowingByCpFw(); + priv_size = rocr_proxy::CreateHwQueuePrivData(&priv_data, queue->context, + FwManagedGfxState, queue->prio); + + D3DKMT_CREATEHWQUEUE createHwQueue = {0}; + createHwQueue.hHwContext = queue->context; + createHwQueue.Flags.DisableGpuTimeout = rocr_proxy::ShouldDisableGpuTimeout(queue->queue_engine, &device_info_); + createHwQueue.pPrivateDriverData = priv_data; + createHwQueue.PrivateDriverDataSize = priv_size; + + NTSTATUS ret = D3DKMTCreateHwQueue(&createHwQueue); + if (ret != STATUS_SUCCESS) { + fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret); + rocr_proxy::DestroyPrivData(priv_data); + return false; + } + + rocr_proxy::DestroyPrivData(priv_data); + + queue->queue = createHwQueue.hHwQueue; + queue->syncobj = createHwQueue.hHwQueueProgressFence; + queue->sync_addr = (uint64_t *)createHwQueue.HwQueueProgressFenceCPUVirtualAddress; + + return true; +} + +bool WDDMDevice::DestroyHwQueue(WDDMQueue *queue) { + D3DKMT_DESTROYHWQUEUE DestroyHwQueue = { + .hHwQueue = queue->queue, + }; + + NTSTATUS ret = D3DKMTDestroyHwQueue(&DestroyHwQueue); + if (ret != STATUS_SUCCESS) { + fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret); + return false; + } + + return true; +} + +bool WDDMDevice::SubmitToHwQueue(WDDMQueue *queue, uint64_t command_addr, + uint64_t command_size, uint64_t fence_value) { + void *priv_data; + int priv_size; + + priv_size = rocr_proxy::CreateSubmitPrivData(&priv_data, queue->queue, command_addr, command_size, true); + + D3DKMT_SUBMITCOMMANDTOHWQUEUE args = {0}; + args.hHwQueue = queue->queue; + args.HwQueueProgressFenceId = fence_value; + args.CommandBuffer = command_addr; + args.CommandLength = command_size; + args.pPrivateDriverData = priv_data; + args.PrivateDriverDataSize = priv_size; + + NTSTATUS ret = D3DKMTSubmitCommandToHwQueue(&args); + if (ret != STATUS_SUCCESS) { + fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret); + rocr_proxy::DestroyPrivData(priv_data); + return false; + } + + rocr_proxy::DestroyPrivData(priv_data); + + return true; +} + +} // namespace core +} // namespace rocr diff --git a/wddm/gpu_memory.cpp b/wddm/gpu_memory.cpp new file mode 100644 index 0000000000..42aeaf68d6 --- /dev/null +++ b/wddm/gpu_memory.cpp @@ -0,0 +1,467 @@ +#include +#include +#include "inc/wddm/gpu_memory.h" +#include "inc/wddm/device.h" +#include "util/utils.h" + +using namespace std; + +namespace rocr { +namespace core { + +size_t GpuMemory::CalcChunkNumbers(gpusize size) { + const auto chunk_size = core::WDDMDevice::GpuMemoryChunkSize; + return (size + chunk_size - 1) / chunk_size; +} + +gpusize GpuMemory::AdjustSize(gpusize size) const { + const auto &device_info = device_->DeviceInfo(); + + if (device_info.enable_big_page_alignment && desc_.domain == rocr_proxy::kLocal) { + uint32_t alignment = device_info.big_page_alignment_size; + // BigPage is only supported for allocations > bigPageMinAlignment. + // Also, if bigPageMinAlignment == 0, BigPage optimization is not supported per KMD. + // We do either LargePage or BigPage alignment, whichever has a higher value. + if ((device_info.hw_big_page_min_alignment_size > 0) && (size > device_info.hw_big_page_min_alignment_size)) { + alignment = std::max(alignment, device_info.hw_big_page_min_alignment_size); + if (size > device_info.hw_big_page_alignment_size) + alignment = std::max(alignment, device_info.hw_big_page_alignment_size); + } + if (alignment > 0) + size = AlignUp(size, alignment); + } else { + const size_t min_size = 4096; + size = AlignUp(size, min_size); + } + return size; +} + +GpuMemory::GpuMemory(WDDMDevice *device) : device_(device) { + num_allocations_ = 0; + alloc_handles_ptr_ = nullptr; + alloc_handle_ = 0; + resource_ = 0; +} + +GpuMemory::~GpuMemory() { + FreeGpuVirtualAddress(GpuAddress(), Size()); + FreePhysicalMemory(); +} + +ErrorCode GpuMemory::Init(const GpuMemoryCreateInfo &create_info) { + desc_.domain = create_info.domain; + desc_.adapter_luid = device_->GetLuid(); + desc_.client_size = create_info.size; + desc_.alignment = create_info.alignment; + desc_.mem_flags = create_info.mem_flags; + desc_.engine_flag = create_info.engine_flag; + desc_.flags.is_virtual = create_info.flags.virtual_alloc; + desc_.flags.is_physical_only = create_info.flags.physical_only; + desc_.flags.is_shared = create_info.flags.interprocess; + desc_.flags.is_locked = create_info.flags.locked; + + desc_.size = AdjustSize(desc_.client_size); + + if (IsUserMemory() || IsSystem()) + desc_.cpu_addr = create_info.user_ptr; + + num_allocations_ = CalcChunkNumbers(Size()); + if (num_allocations_ == 1) + alloc_handles_ptr_ = &alloc_handle_; + else + alloc_handles_ptr_ = new WinAllocationHandle[num_allocations_]; + + memset(alloc_handles_ptr_, 0, num_allocations_ * sizeof(WinAllocationHandle)); + + auto code = ErrorCode::Success; + + if (IsPhysicalOnly()) { + code = CreatePhysicalMemory(); + return code; + } + + code = ReserveGpuVirtualAddress(create_info.va_hint, Size(), create_info.alignment); + if (IsVirtual() || (code != ErrorCode::Success)) + return code; + + bool physical_created = false; + + auto guard = MakeScopeGuard([this, &physical_created, &code]() { + if (code != ErrorCode::Success) { + + if (physical_created) { + FreePhysicalMemory(); + } + FreeGpuVirtualAddress(GpuAddress(), Size()); + } + }); + (void)guard; + + code = CreatePhysicalMemory(); + if (code != ErrorCode::Success) + return code; + + physical_created = true; + + code = MapGpuVirtualAddress(GpuAddress(), Size()); + if (code != ErrorCode::Success) + return code; + + code = MakeResident(); + if (code != ErrorCode::Success) + return code; + + if (!GetDevice()->WaitOnPagingFenceFromCpu()) + code = ErrorCode::Unknown; + + return code; +} + +ErrorCode GpuMemory::UnmapGpuVirtualAddress(const gpusize addr, const gpusize size, gpusize offset) { + auto code = ErrorCode::Success; + size_t i = 0; + auto map_addr = addr; + auto map_size = size; + + while (offset >= core::WDDMDevice::GpuMemoryChunkSize) { + offset -= core::WDDMDevice::GpuMemoryChunkSize; + i += 1; + } + + while (map_size > 0) { + auto block_size = std::min(map_size, core::WDDMDevice::GpuMemoryChunkSize); + + D3DDDI_MAPGPUVIRTUALADDRESS args{}; + + args.hPagingQueue = device_->PagingQueue(); + args.BaseAddress = map_addr; + args.hAllocation = GetAllocationHandle(i); + args.SizeInPages = block_size / 0x1000; + args.Protection.NoAccess = 1; + + code = thunk::MapGpuVirtualAddress(&args); + + if (code == ErrorCode::NotReady) + device_->UpdatePageFence(args.PagingFenceValue); + else if (code != ErrorCode::Success) + break; + + map_addr += block_size; + map_size -= block_size; + offset = 0; // reset second unmapped allocation offset to zero + i += 1; + } + + return code; +} + +ErrorCode GpuMemory::MapGpuVirtualAddress(const gpusize addr, const gpusize size, gpusize offset) { + + auto code = ErrorCode::Success; + size_t i = 0; + auto map_addr = addr; + auto map_size = size; + const size_t _4K = 0x1000; + + while (offset >= core::WDDMDevice::GpuMemoryChunkSize) { + offset -= core::WDDMDevice::GpuMemoryChunkSize; + i += 1; + } + const size_t first_chunk = i; + const auto first_chunk_offset = offset; + /* Found two limitation for local vram: + * 1. invisible vram va has to be 64K aligned, otherwise map gpu va fail + * 2. visible vram can not be cpu mapped when command submission or after gpu mapped + */ + while (map_size > 0) { + auto block_size = std::min(map_size, core::WDDMDevice::GpuMemoryChunkSize); + + D3DDDI_MAPGPUVIRTUALADDRESS args{}; + + args.hPagingQueue = device_->PagingQueue(); + args.BaseAddress = map_addr; + args.hAllocation = GetAllocationHandle(i); + args.OffsetInPages = offset / _4K; + args.SizeInPages = block_size / _4K; + args.Protection.Write = 1; + + code = thunk::MapGpuVirtualAddress(&args); + + if (code != ErrorCode::Success) { + if (code == ErrorCode::NotReady) { + const uint64_t fence_value = args.PagingFenceValue; + device_->UpdatePageFence(fence_value); + code = ErrorCode::Success; + } else + break; + } + + map_addr += block_size; + map_size -= block_size; + offset = 0; // reset second mapped allocation offset to zero + i++; + } + + if (code != ErrorCode::Success) { + // Map failed, unmap partial mapped block + offset = first_chunk_offset; + map_addr = addr; + map_size = size; + for (size_t j = first_chunk; j < i; j++) { + auto block_size = std::min(map_size, core::WDDMDevice::GpuMemoryChunkSize); + + D3DDDI_MAPGPUVIRTUALADDRESS args{}; + + args.hPagingQueue = device_->PagingQueue(); + args.BaseAddress = map_addr; + args.hAllocation = 0; + args.OffsetInPages = offset / _4K; + args.SizeInPages = block_size / _4K; + args.Protection.NoAccess = 1; + + auto unmap_code = thunk::MapGpuVirtualAddress(&args); + if (unmap_code == ErrorCode::NotReady) + device_->UpdatePageFence(args.PagingFenceValue); + + map_addr += block_size; + map_size -= block_size; + } + } + return code; +} + +ErrorCode GpuMemory::ReserveGpuVirtualAddress(gpusize base_virt_addr, gpusize size, gpusize alignment) { + gpusize gpu_virt_addr = 0; + auto status = device_->ReserveGpuVirtualAddress(desc_.domain, base_virt_addr, size, &gpu_virt_addr, alignment, + desc_.flags.is_locked); + if (status == ErrorCode::Success) { + desc_.gpu_addr = gpu_virt_addr; + + if (IsSystem()) + desc_.cpu_addr = reinterpret_cast(desc_.gpu_addr); + } + return status; +} + +ErrorCode GpuMemory::FreeGpuVirtualAddress(gpusize base_addr, gpusize size) { + return base_addr != 0 ? + device_->FreeGpuVirtualAddress(desc_.domain, base_addr, size) : + ErrorCode::Success; +} + +ErrorCode GpuMemory::CreatePhysicalMemory() { + + assert(!IsVirtual() && NumChunks() > 0); + + const auto num_allocations = NumChunks(); + void *priv_drv_data; + void *alloc_priv; + int priv_drv_data_size; + int alloc_priv_data_size; + + if (!rocr_proxy::CreatePrivateAllocInfo(NumChunks(), &priv_drv_data, &alloc_priv, + &priv_drv_data_size, &alloc_priv_data_size)) + return ErrorCode::OutOfMemory; + + auto alloc_info = reinterpret_cast( + static_cast(priv_drv_data) + priv_drv_data_size * num_allocations); + + size_t size = desc_.size; + uint64_t addr = desc_.gpu_addr; + char *cpu_addr = static_cast(desc_.cpu_addr); + const auto &device_info = GetDevice()->DeviceInfo(); + + for (size_t i = 0; i < num_allocations; i++) { + + void* priv_data = (void*)((char*)priv_drv_data + priv_drv_data_size * i); + size_t block_size = std::min(size, core::WDDMDevice::GpuMemoryChunkSize); + + if (IsUserMemory() || IsSystem()) { + rocr_proxy::SetAllocationInfo(priv_data, block_size, desc_.domain, 0, desc_.mem_flags, desc_.engine_flag, device_info); + alloc_info[i].pSystemMem = static_cast(cpu_addr); + cpu_addr += block_size; + } else { + rocr_proxy::SetAllocationInfo(priv_data, block_size, desc_.domain, addr, desc_.mem_flags, desc_.engine_flag, device_info); + } + + size -= block_size; + addr += block_size; + + alloc_info[i].pPrivateDriverData = priv_data; + alloc_info[i].PrivateDriverDataSize = priv_drv_data_size; + alloc_info[i].VidPnSourceId = D3DDDI_ID_UNINITIALIZED; + } + + D3DKMT_CREATEALLOCATION args = {}; + args.hDevice = device_->DeviceHandle(); + args.pPrivateDriverData = alloc_priv; + args.PrivateDriverDataSize = alloc_priv_data_size; + args.NumAllocations = num_allocations; + args.pAllocationInfo2 = alloc_info; + + SharedHandleInfo shared_info; + if (IsShared()) { + shared_info.size = desc_.size; + shared_info.client_size = desc_.client_size; + shared_info.domain = desc_.domain; + shared_info.adapter_luid = desc_.adapter_luid; + shared_info.flags = reinterpret_cast(desc_.flags.reserved); + shared_info.mem_flags = desc_.mem_flags; + args.pPrivateRuntimeData = &shared_info; + args.PrivateRuntimeDataSize = sizeof(shared_info); + args.Flags.NtSecuritySharing = 1; + args.Flags.CreateShared = 1; + args.Flags.CreateResource = 1; + } + + auto status = thunk::CreateAllocation(&args); + if (status == ErrorCode::Success) { + for (size_t i = 0; i < num_allocations; i++) + alloc_handles_ptr_[i] = alloc_info[i].hAllocation; + + resource_ = args.hResource; + } + rocr_proxy::DestroyPrivateAllocInfo(priv_drv_data, alloc_priv); + return status; +} + +ErrorCode GpuMemory::FreePhysicalMemory() { + auto code = ErrorCode::Success; + + if (alloc_handles_ptr_ == nullptr || (NumChunks() == 1 && *alloc_handles_ptr_ == 0)) + return code; + + code = thunk::DestroyAllocation(device_->DeviceHandle(), + resource_, + NumChunks(), + alloc_handles_ptr_); + if (NumChunks() > 1) + delete[] alloc_handles_ptr_; + + alloc_handles_ptr_ = nullptr; + return code; +} + +ErrorCode GpuMemory::MakeResident() { + + D3DDDI_MAKERESIDENT args = {}; + args.hPagingQueue = device_->PagingQueue(); + args.NumAllocations = NumChunks(); + args.AllocationList = alloc_handles_ptr_; + args.Flags.CantTrimFurther = 1; + + auto code = thunk::MakeResident(&args); + if (code == ErrorCode::NotReady) { + const auto fence_value = args.PagingFenceValue; + device_->UpdatePageFence(fence_value); + code = ErrorCode::Success; + } + return code; +} + +ErrorCode GpuMemory::Evict() { + + D3DKMT_EVICT args = {}; + args.hDevice = device_->DeviceHandle(); + args.NumAllocations = NumChunks(); + args.AllocationList = alloc_handles_ptr_; + + return thunk::Evict(&args); +} + +ErrorCode GpuMemory::ExportPhysicalHandle(int* dmabuf_fd, uint32_t flags) { + if (IsShared()) + return thunk::ShareObjects(num_allocations_, resource_, flags, dmabuf_fd); + else + return ErrorCode::UnSupported; +} + + +ErrorCode GpuMemory::ImportPhysicalHandle(int dmabuf_fd) { + D3DKMT_QUERYRESOURCEINFOFROMNTHANDLE query_args; + + if (dmabuf_fd <= 0) + return ErrorCode::InvalidateParams; + + memset(&query_args, 0, sizeof(query_args)); + query_args.hDevice = device_->DeviceHandle(); + query_args.hNtHandle = reinterpret_cast(dmabuf_fd); + auto ret = thunk::QueryResourceInfoFromNtHandle(&query_args); + if (ret != ErrorCode::Success) { + debug_print("%s query resource info from nt handle failed %d\n", __FUNCTION__, static_cast(ret)); + return ErrorCode::InvalidateParams; + } + debug_print("wsl-rocr: import from nt handle %d, get allocation number %d," + " runtime data size %" PRIx64 " total driver data size %" PRIx64 "resource data size=%" PRIx64 "\n", + dmabuf_fd, + query_args.NumAllocations, + query_args.PrivateRuntimeDataSize, + query_args.TotalPrivateDriverDataSize, + query_args.ResourcePrivateDriverDataSize); + + SharedHandleInfo shared_info; + if(sizeof(shared_info) != query_args.PrivateRuntimeDataSize) { + debug_print("%s shared hanle info size mismatch:%d vs %ld\n", + __FUNCTION__, query_args.PrivateRuntimeDataSize, sizeof(shared_info)); + return ErrorCode::UnSupported; + } + + uint32_t total_size = query_args.NumAllocations * sizeof(D3DDDI_OPENALLOCATIONINFO2) + + query_args.TotalPrivateDriverDataSize + + query_args.ResourcePrivateDriverDataSize; + D3DDDI_OPENALLOCATIONINFO2 *open_info = + reinterpret_cast (calloc(1, total_size)); + if (!open_info) { + debug_print("%s alloc open_info failed, NumAllocations:%d\n", + __FUNCTION__, query_args.NumAllocations); + return ErrorCode::OutOfMemory; + } + + alloc_handles_ptr_ = new WinAllocationHandle[query_args.NumAllocations]; + + D3DKMT_OPENRESOURCEFROMNTHANDLE open_args; + memset(&open_args, 0, sizeof(open_args)); + open_args.hDevice = query_args.hDevice; + open_args.hNtHandle = query_args.hNtHandle; + open_args.NumAllocations = query_args.NumAllocations; + open_args.pOpenAllocationInfo2 = open_info; + open_args.TotalPrivateDriverDataBufferSize = query_args.TotalPrivateDriverDataSize; + open_args.pTotalPrivateDriverDataBuffer = reinterpret_cast + (open_args.pOpenAllocationInfo2 + open_args.NumAllocations); + open_args.ResourcePrivateDriverDataSize = query_args.ResourcePrivateDriverDataSize; + open_args.pResourcePrivateDriverData = reinterpret_cast + (((uint64_t)open_args.pTotalPrivateDriverDataBuffer) + + open_args.TotalPrivateDriverDataBufferSize); + open_args.PrivateRuntimeDataSize = query_args.PrivateRuntimeDataSize; + open_args.pPrivateRuntimeData = reinterpret_cast (&shared_info); + + ret = thunk::OpenResourceFromNtHandle(&open_args); + if (ret != ErrorCode::Success) { + ret = ErrorCode::InvalidateParams; + debug_print("%s open resource failed %d\n", __FUNCTION__, static_cast(ret)); + goto err_out; + } + + desc_.size = shared_info.size; + desc_.client_size = shared_info.client_size; + desc_.domain = shared_info.domain; + desc_.flags.reserved = shared_info.flags; + desc_.mem_flags = shared_info.mem_flags; + desc_.adapter_luid = shared_info.adapter_luid; + resource_ = open_args.hResource; + num_allocations_ = open_args.NumAllocations; + for (int i = 0; i < num_allocations_; i++) + alloc_handles_ptr_[i] = open_info[i].hAllocation; + + free(open_info); + return ErrorCode::Success; + +err_out: + delete[] alloc_handles_ptr_; + alloc_handles_ptr_ = nullptr; + free(open_info); + return ret; +} + +} // namespace code +} // namespace rocr diff --git a/wddm/queue.cpp b/wddm/queue.cpp new file mode 100644 index 0000000000..e6856fbd75 --- /dev/null +++ b/wddm/queue.cpp @@ -0,0 +1,989 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2020, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +#include +#include +#include + +#include "inc/wddm/queue.h" +#include "inc/registers.h" +#include "libhsakmt.h" + +namespace rocr { +namespace core { + +hsa_status_t WDDMQueue::SwsInit(void) { + if (!device->CreateSyncobj(&syncobj, &sync_addr)) + return HSA_STATUS_ERROR; + + if (device->AllocUserQueueMemFromUMD()) { + + GpuMemory *gpu_mem = nullptr; + GpuMemoryCreateInfo create_info{}; + + create_info.domain = rocr_proxy::kUserQueue; + create_info.size = device->GetSwsQueueSize(); + create_info.engine_flag = rocr_proxy::QueueEngine2EngineFlag(queue_engine); + + auto code = device->CreateGpuMemory(create_info, &gpu_mem); + if (code != ErrorCode::Success) { + device->DestroySyncobj(syncobj); + return HSA_STATUS_ERROR; + } + + queue_mem = gpu_mem->GetGpuMemoryHandle(); + queue = gpu_mem->GetAllocationHandle(0); + } + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t WDDMQueue::SwsFini(void) { + device->DestroySyncobj(syncobj); + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t WDDMQueue::SwsSubmit(uint64_t command_addr, + uint64_t command_size, + uint64_t fence_value) { + if (!device->SubmitToSwQueue(this, command_addr, command_size, fence_value)) + return HSA_STATUS_ERROR; + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t WDDMQueue::HwsInit(void) { + if (!device->CreateHwQueue(this)) + return HSA_STATUS_ERROR; + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t WDDMQueue::HwsFini(void) { + if (!device->DestroyHwQueue(this)) + return HSA_STATUS_ERROR; + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t WDDMQueue::HwsSubmit(uint64_t command_addr, + uint64_t command_size, + uint64_t fence_value) { + if (!device->SubmitToHwQueue(this, command_addr, command_size, fence_value)) + return HSA_STATUS_ERROR; + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t WDDMQueue::SetPriority(hsa_amd_queue_priority_t priority) { + if (!use_hws) + return HSA_STATUS_SUCCESS; + + rocr_proxy::SchedLevel new_prio = ConvertSchedLevel(priority); + if (prio == new_prio) + return HSA_STATUS_SUCCESS; + + debug_print("set prio %d -> %d\n", prio, new_prio); + device->DestroyHwQueue(this); + + prio = new_prio; + return HwsInit(); +} + +extern "C" void (*fn_hsa_signal_store_screlease)(hsa_signal_t hsa_signal, hsa_signal_value_t value); +void ComputeQueue::HandleError(hsa_status_t status) { + hsa_signal_t sig = amd_queue_rocr_->queue_inactive_signal; + hsa_signal_value_t val = -1; + + struct queue_error_t { + uint32_t code; + hsa_status_t status; + }; + static const queue_error_t QueueErrors[] = { + {2, HSA_STATUS_ERROR_INCOMPATIBLE_ARGUMENTS}, + {4, HSA_STATUS_ERROR_INVALID_ALLOCATION}, + {8, HSA_STATUS_ERROR_INVALID_CODE_OBJECT}, + //{16, HSA_STATUS_ERROR_INVALID_ARGUMENT}, + {32, HSA_STATUS_ERROR_INVALID_PACKET_FORMAT}, + {64, HSA_STATUS_ERROR_INVALID_ARGUMENT}, + //{128, HSA_STATUS_ERROR_OUT_OF_REGISTERS}, + //{0x20000000, HSA_STATUS_ERROR_MEMORY_APERTURE_VIOLATION}, + //{0x40000000, HSA_STATUS_ERROR_ILLEGAL_INSTRUCTION}, + {0x80000000, HSA_STATUS_ERROR_EXCEPTION}, + }; + for (std::size_t i = 0; i < sizeof(QueueErrors) / sizeof(QueueErrors[0]); ++i) { + if (QueueErrors[i].status == status) { + val = QueueErrors[i].code; + debug_print("error %d, sig_val %d\n", status, val); + break; + } + } + + if (sig.handle) { + fn_hsa_signal_store_screlease(sig, val); + } + if (error_code_) { + error_code_->store(val, std::memory_order_release); + } +} + +void ComputeQueue::AqlToPm4Thread(ComputeQueue *queue) { + + ComputeQueue *cq = queue; + // This timing system is used for sleeping this Thread + // when one packet is invalid for about 2 seconds. + std::chrono::steady_clock::time_point start_time, time; + // Set the polling timeout value for 2 seconds + const std::chrono::milliseconds kMaxElapsed(2000); + uint64_t current_position = cq->GetAqlWriteIndex(); + bool sleep = false; + start_time = std::chrono::steady_clock::now(); + + while (true) { + if (!cq->IsInvalidPacket()) { + hsa_status_t status = cq->Process(); + if (status != HSA_STATUS_SUCCESS) { + fprintf(stderr, "process compute queue fail status = %08x\n", status); + queue->HandleError(status); + break; + } + } + + std::unique_lock lock(queue->thread_cond_lock_); + if (current_position == cq->GetAqlWriteIndex()) { + time = std::chrono::steady_clock::now(); + if (time - start_time > kMaxElapsed) + sleep = true; + } else { + start_time = std::chrono::steady_clock::now(); + current_position = cq->GetAqlWriteIndex(); + sleep = false; + } + // CPU wait for valid packet + if (cq->GetRingWptr()->load() <= cq->GetRingRptr()->load() || + (sleep && cq->IsInvalidPacket())) { + if (queue->thread_stop_) + break; + debug_print("wait %p wptr=%" PRIx64 " rptr=%" PRIx64 "\n", + queue->ring, cq->GetRingWptr()->load(), cq->GetRingRptr()->load()); + queue->thread_cond_.wait(lock); + } + } + + debug_print("aql to pm4 thread %p exit\n", queue->ring); +} + +ComputeQueue::ComputeQueue(WDDMDevice *device, + void *ring, + uint64_t ring_size, + std::atomic *ring_wptr, + std::atomic *ring_rptr, + volatile int64_t *error_addr, + uint32_t cmdbuf_size, + uint32_t engine, + bool use_hws) : + WDDMQueue(device, cmdbuf_size, engine, use_hws), + ring(ring), + ring_size(ring_size), + ring_wptr(ring_wptr), + ring_rptr(ring_rptr), + error_code_(reinterpret_cast*>(error_addr)), + ib_start_addr(0), + ib_size(0), + sync_point(0), + cmdbuf_aql_frame_write_index(0), + cmdbuf_aql_frame_size(0), + needs_barrier(true), + ready_to_submit(false), + thread_stop_(false), + scratch_waves_(device->MaxScratchSlotsPerCu() * device->ComputeUnitCount()), + scratch_size_per_wave_(0), + scratch_size_(0), + scratch_base_(nullptr) { + + bool ret = device->CreateQueue(this); + assert(ret); + + GpuMemoryCreateInfo create_info{}; + create_info.size = PAGE_SIZE; + create_info.domain = rocr_proxy::kSystem; + GpuMemory *gpu_mem = nullptr; + auto code = device->CreateGpuMemory(create_info, &gpu_mem); + assert(code == ErrorCode::Success); + amd_queue_mem_ = gpu_mem->GetGpuMemoryHandle(); + amd_queue_ = reinterpret_cast(gpu_mem->GpuAddress()); + + aql_to_pm4_thread_ = std::thread(AqlToPm4Thread, this); + amd_queue_rocr_ = (amd_queue_t*)((char*)ring_rptr - offsetof(amd_queue_t, read_dispatch_id)); +} + +ComputeQueue::~ComputeQueue() { + thread_cond_lock_.lock(); + thread_stop_ = true; + thread_cond_lock_.unlock(); + thread_cond_.notify_one(); + aql_to_pm4_thread_.join(); + + //doorbell_signal_->Release(); + + device->DestroyQueue(this); + + if (scratch_base_) { + auto scratch_gpu_mem = GpuMemory::Convert(scratch_mem_); + delete scratch_gpu_mem; + } + + auto amd_queue_gpu_mem = GpuMemory::Convert(amd_queue_mem_); + delete amd_queue_gpu_mem; +} + +void ComputeQueue::InitScratchSRD() { + // Populate scratch resource descriptor + SQ_BUF_RSRC_WORD0 srd0; + + uintptr_t scratch_base = uintptr_t(scratch_base_); + srd0.bits.BASE_ADDRESS = scratch_base; + + uint32_t srd1_u32; + + if (device->Major() < 11) { + SQ_BUF_RSRC_WORD1 srd1; + + srd1.bits.BASE_ADDRESS_HI = scratch_base >> 32; + srd1.bits.STRIDE = 0; + srd1.bits.CACHE_SWIZZLE = 0; + srd1.bits.SWIZZLE_ENABLE = 1; + + srd1_u32 = srd1.u32All; + } else { + SQ_BUF_RSRC_WORD1_GFX11 srd1; + + srd1.bits.BASE_ADDRESS_HI = scratch_base >> 32; + srd1.bits.STRIDE = 0; + srd1.bits.SWIZZLE_ENABLE = 1; + + srd1_u32 = srd1.u32All; + } + + SQ_BUF_RSRC_WORD2 srd2; + + srd2.bits.NUM_RECORDS = scratch_size_; + + uint32_t srd3_u32; + + if (device->Major() < 10) { + SQ_BUF_RSRC_WORD3 srd3; + + srd3.bits.DST_SEL_X = SQ_SEL_X; + srd3.bits.DST_SEL_Y = SQ_SEL_Y; + srd3.bits.DST_SEL_Z = SQ_SEL_Z; + srd3.bits.DST_SEL_W = SQ_SEL_W; + srd3.bits.NUM_FORMAT = BUF_NUM_FORMAT_UINT; + srd3.bits.DATA_FORMAT = BUF_DATA_FORMAT_32; + srd3.bits.ELEMENT_SIZE = 1; // 4 + srd3.bits.INDEX_STRIDE = 3; // 64 + srd3.bits.ADD_TID_ENABLE = 1; + srd3.bits.ATC__CI__VI = 0; + srd3.bits.HASH_ENABLE = 0; + srd3.bits.HEAP = 0; + srd3.bits.MTYPE__CI__VI = 0; + srd3.bits.TYPE = SQ_RSRC_BUF; + + srd3_u32 = srd3.u32All; + } else if (device->Major() == 10) { + SQ_BUF_RSRC_WORD3_GFX10 srd3; + + srd3.bits.DST_SEL_X = SQ_SEL_X; + srd3.bits.DST_SEL_Y = SQ_SEL_Y; + srd3.bits.DST_SEL_Z = SQ_SEL_Z; + srd3.bits.DST_SEL_W = SQ_SEL_W; + srd3.bits.FORMAT = BUF_FORMAT_32_UINT; + srd3.bits.RESERVED1 = 0; + srd3.bits.INDEX_STRIDE = 0; // filled in by CP + srd3.bits.ADD_TID_ENABLE = 1; + srd3.bits.RESOURCE_LEVEL = 1; + srd3.bits.RESERVED2 = 0; + srd3.bits.OOB_SELECT = 2; // no bounds check in swizzle mode + srd3.bits.TYPE = SQ_RSRC_BUF; + + srd3_u32 = srd3.u32All; + } else if (device->Major() == 11) { + SQ_BUF_RSRC_WORD3_GFX11 srd3; + + srd3.bits.DST_SEL_X = SQ_SEL_X; + srd3.bits.DST_SEL_Y = SQ_SEL_Y; + srd3.bits.DST_SEL_Z = SQ_SEL_Z; + srd3.bits.DST_SEL_W = SQ_SEL_W; + srd3.bits.FORMAT = BUF_FORMAT_32_UINT; + srd3.bits.RESERVED1 = 0; + srd3.bits.INDEX_STRIDE = 0; // filled in by CP + srd3.bits.ADD_TID_ENABLE = 1; + srd3.bits.RESERVED2 = 0; + srd3.bits.OOB_SELECT = 2; // no bounds check in swizzle mode + srd3.bits.TYPE = SQ_RSRC_BUF; + + srd3_u32 = srd3.u32All; + } else { + SQ_BUF_RSRC_WORD3_GFX12 srd3; + srd3.bits.DST_SEL_X = SQ_SEL_X; + srd3.bits.DST_SEL_Y = SQ_SEL_Y; + srd3.bits.DST_SEL_Z = SQ_SEL_Z; + srd3.bits.DST_SEL_W = SQ_SEL_W; + srd3.bits.FORMAT = BUF_FORMAT_32_UINT; + srd3.bits.RESERVED1 = 0; + srd3.bits.INDEX_STRIDE = 0; // filled in by CP + srd3.bits.ADD_TID_ENABLE = 1; + srd3.bits.WRITE_COMPRESS_ENABLE = 0; + srd3.bits.COMPRESSION_EN = 0; + srd3.bits.COMPRESSION_ACCESS_MODE = 0; + srd3.bits.OOB_SELECT = 2; // no bounds check in swizzle mode + srd3.bits.TYPE = SQ_RSRC_BUF; + + srd3_u32 = srd3.u32All; + } + + // Update Queue's Scratch descriptor's property + amd_queue_->scratch_resource_descriptor[0] = srd0.u32All; + amd_queue_->scratch_resource_descriptor[1] = srd1_u32; + amd_queue_->scratch_resource_descriptor[2] = srd2.u32All; + amd_queue_->scratch_resource_descriptor[3] = srd3_u32; + + // Populate flat scratch parameters in amd_queue_. + amd_queue_->scratch_backing_memory_location = scratch_base; + amd_queue_->scratch_backing_memory_byte_size = scratch_size_; + + // For backwards compatibility this field records the per-lane scratch + // for a 64 lane wavefront. If scratch was allocated for 32 lane waves + // then the effective size for a 64 lane wave is halved. + amd_queue_->scratch_wave64_lane_byte_size = scratch_size_per_wave_ / 64; + + if (device->Major() < 11) { + COMPUTE_TMPRING_SIZE tmpring_size; + tmpring_size.bits.WAVESIZE = scratch_size_per_wave_ / 1024; + tmpring_size.bits.WAVES = scratch_waves_; + + amd_queue_->compute_tmpring_size = tmpring_size.u32All; + } else if (device->Major() == 11) { + COMPUTE_TMPRING_SIZE_GFX11 tmpring_size; + tmpring_size.bits.WAVESIZE = scratch_size_per_wave_ >> 8; + tmpring_size.bits.WAVES = scratch_waves_ / device->NumShaderEngine(); + + amd_queue_->compute_tmpring_size = tmpring_size.u32All; + } else { + COMPUTE_TMPRING_SIZE_GFX12 tmpring_size = {}; + tmpring_size.bits.WAVESIZE = scratch_size_per_wave_ >> 8; + tmpring_size.bits.WAVES = scratch_waves_ / device->NumShaderEngine(); + + amd_queue_->compute_tmpring_size = tmpring_size.u32All; + } + + return; +} + +bool ComputeQueue::UpdateScratch(uint32_t private_segment_size, bool wave32) { + const uint32_t wavefront = wave32 ? 32 : 64; + const uint32_t alignment = 1024 / wavefront; + private_segment_size = AlignUp(private_segment_size, alignment); + + uint32_t scratch_size_per_wave = private_segment_size * wavefront; + uint32_t scratch_size = scratch_size_per_wave * scratch_waves_; + + if (scratch_size_ >= scratch_size) + return true; + + debug_print("need realloc scratch buffer, size %x -> %x\n", + scratch_size_, scratch_size); + + GpuMemoryCreateInfo create_info{}; + create_info.size = scratch_size; + create_info.domain = rocr_proxy::kLocal; + GpuMemory *gpu_mem = nullptr; + auto code = device->CreateGpuMemory(create_info, &gpu_mem); + if (code != ErrorCode::Success) + return false; + + if (scratch_base_) { + auto scratch_gpu_mem = GpuMemory::Convert(scratch_mem_); + delete scratch_gpu_mem; + } + + scratch_size_per_wave_ = scratch_size_per_wave; + scratch_size_ = scratch_size; + scratch_base_ = reinterpret_cast(gpu_mem->GpuAddress()); + scratch_mem_ = gpu_mem->GetGpuMemoryHandle(); + + InitScratchSRD(); + return true; +} + +bool ComputeQueue::RelocateCmdbufScratchBase(uint64_t addr) { + if (scratch_base_offset_array_.empty()) + return true; + + for (size_t i = 0; i < scratch_base_offset_array_.size(); i++) { + uint32_t *p_compute_user_data = + reinterpret_cast(addr + scratch_base_offset_array_[i]); + if (device->Major() >= 11) { + p_compute_user_data[0] = Ptr48Low32(scratch_base_); + p_compute_user_data[1] = Ptr48High8(scratch_base_); + } else { + p_compute_user_data[0] = PtrLow32(scratch_base_); + p_compute_user_data[1] = (p_compute_user_data[1] & 0xffff0000) | PtrHigh32(scratch_base_); + } + } + scratch_base_offset_array_.clear(); + + return true; +} + +uint32_t ComputeQueue::UpdateIndexStride(uint32_t srd, bool wave32) { + + assert(device->Major() < 13); + + if (device->Major() == 10) { + SQ_BUF_RSRC_WORD3_GFX10 srd3; + + srd3.u32All = srd; + srd3.bits.INDEX_STRIDE = wave32 ? 2 : 3; + + return srd3.u32All; + } else if (device->Major() == 11) { + SQ_BUF_RSRC_WORD3_GFX11 srd3; + + srd3.u32All = srd; + srd3.bits.INDEX_STRIDE = wave32 ? 2 : 3; + + return srd3.u32All; + } else if (device->Major() == 12) { + SQ_BUF_RSRC_WORD3_GFX12 srd3; + + srd3.u32All = srd; + srd3.bits.INDEX_STRIDE = wave32 ? 2 : 3; + + return srd3.u32All; + } + + return srd; +} + +extern "C" hsa_status_t (*fn_hsa_ven_amd_loader_query_host_address)( + const void *device_address, + const void **host_address); +uint64_t ComputeQueue::GetKernelObjAddr(uint64_t addr) const { +//TODO: convert dev_addr to host_addr + uint64_t host_addr = 0; + auto ret = fn_hsa_ven_amd_loader_query_host_address(reinterpret_cast(addr), + reinterpret_cast(&host_addr)); + if (ret == HSA_STATUS_ERROR_INVALID_ARGUMENT) { + return NULL; + } + + return host_addr; +} + +void ComputeQueue::RingDoorbell() { + thread_cond_lock_.lock(); + thread_cond_lock_.unlock(); + debug_print("notify %p wptr=%" PRIx64 " rptr=%" PRIx64 "\n", + ring, GetRingWptr()->load(), GetRingRptr()->load()); + thread_cond_.notify_one(); +} + +hsa_status_t ComputeQueue::Init(void) { + hsa_status_t ret = use_hws ? HwsInit() : SwsInit(); + if (ret) + return ret; + + ib_start_addr = cmdbuf_addr; + cmdbuf_aql_frame_size = device->GetAqlFrameSize(); + + return ret; +} + +hsa_status_t ComputeQueue::Fini(void) { + return use_hws ? HwsFini() : SwsFini(); +} + +hsa_status_t ComputeQueue::PreSubmit(void) { + if (!device->WaitPagingFence(this)) + return HSA_STATUS_ERROR; + + RelocateCmdbufScratchBase(ib_start_addr); + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t ComputeQueue::EndSubmit(void) { + // record last submitted cmdbuf_aql_frame_write_index to see if GPU is hungry + sync_point = cmdbuf_aql_frame_write_index; + + ib_start_addr = cmdbuf_addr + + (cmdbuf_aql_frame_write_index % WDDMDevice::GetAqlFrameNum()) * + cmdbuf_aql_frame_size; + ib_size = 0; + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t ComputeQueue::Submit(void) { + hsa_status_t ret = PreSubmit(); + if (ret) + return HSA_STATUS_ERROR; + + ret = use_hws ? + HwsSubmit(ib_start_addr, ib_size, cmdbuf_aql_frame_write_index) : + SwsSubmit(ib_start_addr, ib_size, cmdbuf_aql_frame_write_index); + if (ret) + return HSA_STATUS_ERROR; + + ret = EndSubmit(); + if (ret) + return HSA_STATUS_ERROR; + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t +ComputeQueue::KernelDispatchAqlToPm4(char *cpu, hsa_kernel_dispatch_packet_t *packet) { + debug_print("queue %p kernel dispatch head=%x setup=%x wx=%x wy=%x wz=%x " + "gx=%x gy=%x gz=%x ps=%x gs=%x ko=%" PRIx64 " ka=%p cs=%" PRIx64 "\n", + ring, packet->header, + packet->setup, packet->workgroup_size_x, packet->workgroup_size_y, + packet->workgroup_size_z, packet->grid_size_x, packet->grid_size_y, + packet->grid_size_z, packet->private_segment_size, + packet->group_segment_size, packet->kernel_object, packet->kernarg_address, + packet->completion_signal.handle); + + if (packet->workgroup_size_x > 1024 || + packet->workgroup_size_y > 1024 || + packet->workgroup_size_z > 1024) + return HSA_STATUS_ERROR_INVALID_ARGUMENT; + + int major = device->Major(); + int i = ib_size; + + const amd_kernel_code_t* kernel_object = + (const amd_kernel_code_t *)GetKernelObjAddr(packet->kernel_object); + if (kernel_object == NULL) { + return HSA_STATUS_ERROR_INVALID_CODE_OBJECT; + } + + void* entry = (void*)(packet->kernel_object + kernel_object->kernel_code_entry_byte_offset); + assert((size_t)entry % AMD_ISA_ALIGN_BYTES == 0); + + debug_print("kernel object property=%x entry=%p lds=%x+%x\n", + kernel_object->kernel_code_properties, entry, + kernel_object->workgroup_group_segment_byte_size, + packet->group_segment_size); + + if (packet->setup == 0 || packet->setup > 3) + return HSA_STATUS_ERROR_INCOMPATIBLE_ARGUMENTS; + if (packet->group_segment_size > device->LdsSize()) + return HSA_STATUS_ERROR_INVALID_ALLOCATION; + + uint32_t lds_blks = device->LdsBlocks(packet); + if (lds_blks > 128) + return HSA_STATUS_ERROR_INVALID_ARGUMENT; + + const bool wave32 = + AMD_HSA_BITS_GET(kernel_object->kernel_code_properties, + AMD_KERNEL_CODE_PROPERTIES_ENABLE_WAVEFRONT_SIZE32); + + assert(packet->private_segment_size >= kernel_object->workitem_private_segment_byte_size); + UpdateScratch(packet->private_segment_size, wave32); + + amd_signal_t *signal = (amd_signal_t *)packet->completion_signal.handle; + + // Record start timestamp when enabling profiling + if (signal && EnableProfiling()) + i += cmd_util.BuildCopyData(&signal->start_ts, cpu + i); + + // Build a barrier packet if it is requested + const bool is_barrier_packet = (packet->header >> HSA_PACKET_HEADER_BARRIER) & 0x1; + if (is_barrier_packet && needs_barrier) + i += cmd_util.BuildBarrier(cpu + i); + + // flush cache + i += cmd_util.BuildAcquireMem(major, cpu + i); + + if (major >= 11) { + AppendCmdbufSratchBaseOffset( + i + offsetof(struct SetScratchTemplate, scratch_lo)); + + i += cmd_util.BuildScratch(ScratchBase(), cpu + i); + i += cmd_util.BuildComputeShaderParams(cpu + i); + } + + struct DispatchInfo info; + info.major = major; + info.pPacket = packet; + info.pEntry = entry; + info.pKernelObject = kernel_object; + info.ldsBlks = lds_blks; + info.pAmdQueue = amd_queue_; + info.wave32 = wave32; + info.srd = UpdateIndexStride( + info.pAmdQueue->scratch_resource_descriptor[3], wave32); + info.pScratchBase = ScratchBase(); + info.scratchSizePerWave = ScratchSizePerWave(); + memset(info.scratchBaseOffset, 0, sizeof(info.scratchBaseOffset)); + info.offsetCnt = 0; + + size_t size; + size = cmd_util.BuildDispatch(&info, cpu + i); + for (int j = 0; j < info.offsetCnt; j++) + AppendCmdbufSratchBaseOffset(i + info.scratchBaseOffset[j]); + i += size; + + needs_barrier = (packet->completion_signal.handle == 0); + + if (signal) { + // wait cs done + i += cmd_util.BuildBarrier(cpu + i); + + // Record end timestamp when enabling profiling + if (EnableProfiling()) + i += cmd_util.BuildCopyData(&signal->end_ts, cpu + i); + + // flush cache + i += cmd_util.BuildAcquireMem(major, cpu + i); + + assert(signal->kind == AMD_SIGNAL_KIND_USER); + uint64_t *signal_addr = (uint64_t *)&signal->value; + debug_print("signal value=%" PRIx64 "\n", signal->value); + + i += cmd_util.BuildAtomicMem(signal_addr, TC_OP_ATOMIC_ADD_RTN_64, cpu + i, cache_policy__mec_atomic_mem__bypass, -1); + } + + // The ring_rptr is used to record pm4 queue rptr value, + // dispatch readptr position, this is used to share rptr with + // aql queue. + i += cmd_util.BuildAtomicMem((uint64_t *)ring_rptr, TC_OP_ATOMIC_ADD_RTN_64, cpu + i); + + ib_size = i; + cmdbuf_aql_frame_write_index++; + packet->header = HSA_PACKET_TYPE_INVALID; + + return HSA_STATUS_SUCCESS; +} + +extern "C" hsa_signal_value_t (*fn_hsa_signal_load_relaxed)( + hsa_signal_t signal); +extern "C" hsa_signal_value_t (*fn_hsa_signal_wait_relaxed)( + hsa_signal_t signal, + hsa_signal_condition_t condition, + hsa_signal_value_t compare_value, + uint64_t timeout_hint, + hsa_wait_state_t wait_state_hint); +hsa_status_t +ComputeQueue::BarrierGenericAqlToPm4(char *cpu, hsa_barrier_and_packet_t *packet, bool is_or) { + debug_print("queue %p %s head=%x dep %" PRIx64 " %" PRIx64 " %" PRIx64 + " %" PRIx64 " %" PRIx64 " cs=%" PRIx64"\n", + ring, is_or ? "or" : "and", + packet->header, packet->dep_signal[0].handle, + packet->dep_signal[1].handle, packet->dep_signal[2].handle, + packet->dep_signal[3].handle, packet->dep_signal[4].handle, + packet->completion_signal.handle); + // fix me: can we use gpu packet? + if (is_or) { + bool unsignaled = true; + hsa_signal_t sig[5]; + int n = 0; + for (int i = 0; i < 5; i++) { + if (packet->dep_signal[i].handle) + sig[n++] = packet->dep_signal[i]; + } + + while (n) { + for (int i = 0; i < n; i++) { + if (!fn_hsa_signal_load_relaxed(sig[i])) { + unsignaled = false; + break; + } + } + if (!unsignaled) + break; + + std::this_thread::sleep_for(std::chrono::microseconds(20)); + } + } else { + for (int i = 0; i < 5; i++) { + if (!packet->dep_signal[i].handle) + continue; + + hsa_signal_value_t value = + fn_hsa_signal_wait_relaxed(packet->dep_signal[i], HSA_SIGNAL_CONDITION_EQ, 0, UINT64_MAX, HSA_WAIT_STATE_BLOCKED); + assert(value == 0); + } + } + + int major = device->Major(); + int i = ib_size; + + if (packet->completion_signal.handle != 0) { + amd_signal_t *signal = (amd_signal_t *)packet->completion_signal.handle; + assert(signal->kind == AMD_SIGNAL_KIND_USER); + uint64_t *signal_addr = (uint64_t *)&signal->value; + debug_print("signal value=%" PRIx64 "\n", signal->value); + + // Record start timestamp when enabling profiling + if (EnableProfiling()) + i += cmd_util.BuildCopyData(&signal->start_ts, cpu + i); + + if (needs_barrier) + i += cmd_util.BuildBarrier(cpu + i); + + needs_barrier = false; + + // Record end timestamp when enabling profiling + if (EnableProfiling()) + i += cmd_util.BuildCopyData(&signal->end_ts, cpu + i); + + // flush cache + i += cmd_util.BuildAcquireMem(major, cpu + i); + + i += cmd_util.BuildAtomicMem(signal_addr, TC_OP_ATOMIC_ADD_RTN_64, cpu + i, cache_policy__mec_atomic_mem__bypass, -1); + } + + // The ring_rptr is used to record pm4 queue rptr value, + // dispatch readptr position, this is used to share rptr with + // aql queue. + i += cmd_util.BuildAtomicMem((uint64_t *)ring_rptr, TC_OP_ATOMIC_ADD_RTN_64, cpu + i); + + ib_size = i; + cmdbuf_aql_frame_write_index++; + packet->header = HSA_PACKET_TYPE_INVALID; + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t ComputeQueue::VendorSpecificAqlToPm4(char *cpu, amd_aql_pm4_ib *packet) { + constexpr uint32_t AMD_AQL_FORMAT_PM4_IB = 0x1; + assert(packet->ven_hdr == AMD_AQL_FORMAT_PM4_IB); + + uint8_t op = (packet->ib_jump_cmd[0] >> PM4_OPCODE_SHIFT) & 0xff; + assert(op == IT_INDIRECT_BUFFER); + uint32_t* pm4_addr = reinterpret_cast((static_cast(packet->ib_jump_cmd[2]) << 32) | (static_cast(packet->ib_jump_cmd[1]) & ~3ull)); + uint32_t pm4_size = packet->ib_jump_cmd[3]&0xfffff; + debug_print("queue %p %s VENDOR_SPECIFIC pkt pm4_addr %p pm4_size %" PRIx64 " cs=%" PRIx64"\n", + ring, vendor_packet_support ? "process" : "skip", pm4_addr, pm4_size, + packet->completion_signal.handle); + for (int i = 0; i < pm4_size; i++) { + debug_print("pm4_addr[%d]=%#x\n", i, pm4_addr[i]); + } + + if (vendor_packet_support) { + int major = device->Major(); + int i = ib_size; + + memcpy(cpu+i, pm4_addr, pm4_size * sizeof(uint32_t)); + i += pm4_size * sizeof(uint32_t); + + if (packet->completion_signal.handle != 0) { + amd_signal_t *signal = (amd_signal_t *)packet->completion_signal.handle; + assert(signal->kind == AMD_SIGNAL_KIND_USER); + uint64_t *signal_addr = (uint64_t *)&signal->value; + debug_print("signal value=%" PRIx64 "\n", signal->value); + + // Record start timestamp when enabling profiling + if (EnableProfiling()) + i += cmd_util.BuildCopyData(&signal->start_ts, cpu + i); + + //if (needs_barrier) + i += cmd_util.BuildBarrier(cpu + i); + + //needs_barrier = false; + + // Record end timestamp when enabling profiling + if (EnableProfiling()) + i += cmd_util.BuildCopyData(&signal->end_ts, cpu + i); + + // flush cache + i += cmd_util.BuildAcquireMem(major, cpu + i); + + i += cmd_util.BuildAtomicMem(signal_addr, TC_OP_ATOMIC_ADD_RTN_64, cpu + i, cache_policy__mec_atomic_mem__bypass, -1); + } + + // The ring_rptr is used to record pm4 queue rptr value, + // dispatch readptr position, this is used to share rptr with + // aql queue. + i += cmd_util.BuildAtomicMem((uint64_t *)ring_rptr, TC_OP_ATOMIC_ADD_RTN_64, cpu + i); + + ib_size = i; + } else { + if (packet->completion_signal.handle != 0) { + fn_hsa_signal_store_screlease(packet->completion_signal, 0); + } + } + + cmdbuf_aql_frame_write_index++; + packet->header = HSA_PACKET_TYPE_INVALID; + return HSA_STATUS_SUCCESS; +} + +hsa_status_t ComputeQueue::SwitchAql2PM4(void) { + + uint16_t *packet = (uint16_t *) ((char *)ring + + (cmdbuf_aql_frame_write_index % ring_size) * 64); + uint16_t header = (*packet >> HSA_PACKET_HEADER_TYPE); + header &= (1 << HSA_PACKET_HEADER_WIDTH_TYPE) - 1; + hsa_kernel_dispatch_packet_t *aql_packet = + (hsa_kernel_dispatch_packet_t *)packet; + hsa_status_t ret; + + switch (header) { + case HSA_PACKET_TYPE_KERNEL_DISPATCH: + ret = KernelDispatchAqlToPm4((char *)ib_start_addr, aql_packet); + if (ret != HSA_STATUS_SUCCESS) + return ret; + + // Stop merging packages util below conditions are met: + // 1) The kernel with completion signal; + // 2) The cmdbuf_aql_frame_write_index reaches the end of cmdbuf + // 3) The queue is empty now, submit the package right now. + if (!(aql_packet->completion_signal.handle) && + (cmdbuf_aql_frame_write_index % WDDMDevice::GetAqlFrameNum()) && + (*sync_addr != sync_point)) + return HSA_STATUS_SUCCESS; + + break; + case HSA_PACKET_TYPE_BARRIER_AND: + BarrierGenericAqlToPm4((char *)ib_start_addr, (hsa_barrier_and_packet_t *)aql_packet); + break; + case HSA_PACKET_TYPE_BARRIER_OR: + BarrierGenericAqlToPm4((char *)ib_start_addr, (hsa_barrier_and_packet_t *)aql_packet, true); + break; + case HSA_PACKET_TYPE_VENDOR_SPECIFIC: + VendorSpecificAqlToPm4((char *)ib_start_addr, (amd_aql_pm4_ib *)aql_packet); + break; + case HSA_PACKET_TYPE_INVALID: + // When packets are submitted out of order, the format field of current AQL packet + // may not have been updated yet and is still INVALID. Return HSA_STATUS_SUCCESS and + // do not process AQL packets before the packet format field is updated. + assert(false && "Should not reach here, HSA_PACKET_TYPE_INVALID has been filtered in upper layer"); + return HSA_STATUS_SUCCESS; + default: + return HSA_STATUS_ERROR_INVALID_PACKET_FORMAT; + } + + ready_to_submit = true; + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t ComputeQueue::Process(void) { + + while (cmdbuf_aql_frame_write_index < ring_wptr->load() && + !IsInvalidPacket()) { + debug_print("process %p wptr=%" PRIx64 " rptr=%" PRIx64 "\n", + ring, ring_wptr->load(), ring_rptr->load()); + + hsa_status_t ret; + + // wait for next few cmdbuf slots to be free + // If wptr catch up the rptr in the cmdbuf, this needs wait for the rptr to free the cmdbuf. + // Here the wptr comes from queue->cmdbuf_aql_frame_write_index, while rptr comes from *queue->sync_addr. + if (*sync_addr + WDDMDevice::GetAqlFrameNum() <= cmdbuf_aql_frame_write_index) { + uint64_t value = cmdbuf_aql_frame_write_index - WDDMDevice::GetAqlFrameNum() + 1; + if (!device->CpuWait(&syncobj, &value, 1, false)) + return HSA_STATUS_ERROR; + } + + ret = SwitchAql2PM4(); + if (ret != HSA_STATUS_SUCCESS) + return ret; + + if (!ready_to_submit) + continue; + + ret = Submit(); + if (ret != HSA_STATUS_SUCCESS) + return ret; + + ready_to_submit = false; + + debug_print("done %p wptr=%" PRIx64 " rptr=%" PRIx64 "\n", + ring, ring_wptr->load(), ring_rptr->load()); + + } + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t SDMAQueue::Init(void) { + hsa_status_t ret = use_hws ? HwsInit() : SwsInit(); + if (ret) + return ret; + + std::memset((char *)cmdbuf_addr, 0, cmdbuf_size); + + return ret; +} + +hsa_status_t SDMAQueue::Fini(void) { + return use_hws ? HwsFini() : SwsFini(); +} + +int SDMAQueue::PreparePacket(uint32_t offset, uint64_t size) { + ib_start_addr = cmdbuf_addr + offset; + ib_size = size; + rptr_next += ib_size; + + return STATUS_SUCCESS; +} + +hsa_status_t SDMAQueue::Submit(void) { + if (!device->WaitPagingFence(this)) + return HSA_STATUS_ERROR; + + int ret = use_hws ? + HwsSubmit(ib_start_addr, ib_size, rptr_next) : + SwsSubmit(ib_start_addr, ib_size, rptr_next); + if (ret) + return HSA_STATUS_ERROR; + + return HSA_STATUS_SUCCESS; +} + +} +} diff --git a/wddm/va_mgr.cpp b/wddm/va_mgr.cpp new file mode 100644 index 0000000000..426eaa2a5f --- /dev/null +++ b/wddm/va_mgr.cpp @@ -0,0 +1,163 @@ +#include +#include +#include +#include "inc/wddm/va_mgr.h" + +using namespace std; + +namespace rocr { +namespace core { + +VaMgr::VaMgr(uint64_t start, uint64_t size, uint64_t min_align) { + min_align_ = min_align; + auto free_it = free_list_.insert(make_pair(size, start)); + frag_map_[start] = make_fragment(free_it, size); +} + +VaMgr::~VaMgr() { + + assert(free_list_.size() == 1); + assert(frag_map_.size() == 1); + + free_list_.clear(); + frag_map_.clear(); +} + +uint64_t VaMgr::Alloc(uint64_t bytes, uint64_t align, uint64_t addr) { + + if (addr > 0 && + (align == 0 || (addr % align) == 0)) { + + lock_guard gard(lock_); + auto frag_it = frag_map_.upper_bound(addr); + assert(frag_it != frag_map_.begin()); + --frag_it; + + while (frag_it != frag_map_.begin()) { + const uint64_t base = frag_it->first; + const uint64_t size = frag_it->second.size; + + // Cannot find free fragment contains the target `addr` + if (bytes > size || addr < base || addr + bytes >= base + size || + !is_free(frag_it->second)) { + --frag_it; + continue; + } else if (addr >= base + size) + break; + + + // Try to allocate target `addr` from this free fragment + auto free_it = frag_it->second.free_list_entry_; + assert(free_it != free_list_.end()); + + free_list_.erase(free_it); + frag_it->second.size = bytes; + set_used(frag_it->second); + + // [base, addr) + if (addr > base) add_free_fragment(addr - base, base); + + // [addr, addr + bytes) is used + + // [addr + bytes, base + size) + if (base + size > addr + bytes) add_free_fragment(base + size - addr - bytes, addr + bytes); + + return addr; + } + } + + // Allocate not fixed address + return AllocImpl(bytes, align); +} + +uint64_t VaMgr::AllocImpl(const uint64_t bytes, const uint64_t align) { + uint64_t addr = 0; + uint64_t align_bytes = bytes; + const int retry = align == 0 ? 0 : 1; + const uint64_t new_align = align == 0 ? min_align_ : AlignUp(align, min_align_); + + lock_guard gard(lock_); + for (int i = 0; i <= retry; i++) { + auto free_it = free_list_.lower_bound(align_bytes); + if (free_it == free_list_.end()) break; + + uint64_t base = free_it->second; + uint64_t size = free_it->first; + + assert(size >= align_bytes); + + auto fragment = frag_map_.find(base); + + assert(fragment != frag_map_.end()); + assert(size == fragment->second.size); + + uint64_t delta = align == 0 ? 0 : base % align; + if (delta == 0) { + // already find aligned address + addr = base; + + free_list_.erase(free_it); + fragment->second.size = bytes; + set_used(fragment->second); + + if (size > bytes) add_free_fragment(size - bytes, base + bytes); + + break; + } else if (i == 0) { + align_bytes += new_align; + continue; + } else { + uint64_t aligned_base = base + align - delta; + addr = aligned_base; + + free_list_.erase(free_it); + + add_used_fragment(bytes, aligned_base); + add_free_fragment(aligned_base - base, base); + + if (size > aligned_base - base + bytes) + add_free_fragment(size - (aligned_base - base) - bytes, aligned_base + bytes); + + break; + } + } + return addr; +} + +void VaMgr::Free(uint64_t addr) { + if (addr == 0) return; + + lock_guard gard(lock_); + auto frag_it = frag_map_.find(addr); + if (frag_it == frag_map_.end() || is_free(frag_it->second)) return; + + uint64_t base = addr; + // Merge lower + if (frag_it != frag_map_.begin()) { + auto lower = frag_it; + --lower; + if (is_free(lower->second)) { + remove_free_list_entry(lower->second); + base -= lower->second.size; + lower->second.size += frag_it->second.size; + frag_map_.erase(frag_it); + frag_it = lower; + } + } + // Merge upper + { + auto upper = frag_it; + ++upper; + if (upper != frag_map_.end() && is_free(upper->second)) { + remove_free_list_entry(upper->second); + frag_it->second.size += upper->second.size; + frag_map_.erase(upper); + } + } + uint64_t size = frag_it->second.size; + auto it = free_list_.insert(make_pair(size, base)); + set_free(frag_it->second, it); +} + +} // namespace core +} // namespace rocr