tiancyin
f8d1663b39
In multi-GPU, handle aperture is shared between all GPUs, not belongs to specific one GPU, so move it from wddm device (which presents a specific GPU) to thunk runtime which has gloable view, can manage handle aperture for all GPUs. Reviewed-by: Flora Cui <flora.cui@amd.com> Signed-off-by: tiancyin <tianci.yin@amd.com>
277 řádky
11 KiB
C++
277 řádky
11 KiB
C++
/*
|
|
* Copyright © 2014 Advanced Micro Devices, Inc.
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person
|
|
* obtaining a copy of this software and associated documentation
|
|
* files (the "Software"), to deal in the Software without
|
|
* restriction, including without limitation the rights to use, copy,
|
|
* modify, merge, publish, distribute, sublicense, and/or sell copies
|
|
* of the Software, and to permit persons to whom the Software is
|
|
* furnished to do so, subject to the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice (including
|
|
* the next paragraph) shall be included in all copies or substantial
|
|
* portions of the Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
|
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
|
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
* DEALINGS IN THE SOFTWARE.
|
|
*/
|
|
|
|
#ifndef LIBHSAKMT_H_INCLUDED
|
|
#define LIBHSAKMT_H_INCLUDED
|
|
|
|
#include <pthread.h>
|
|
#include <stdint.h>
|
|
#include <limits.h>
|
|
#include "hsakmt/hsakmt.h"
|
|
#include "hsakmt/hsakmt_drm.h"
|
|
|
|
#include "impl/wddm/va_mgr.h"
|
|
#include "impl/wddm/types.h"
|
|
#include "impl/wddm/device.h"
|
|
|
|
wsl::thunk::WDDMDevice* get_wddmdev(uint32_t node_id);
|
|
uint32_t get_num_wddmdev();
|
|
wsl::thunk::GpuMemory *get_gpu_mem(void *MemoryAddress);
|
|
|
|
#define HSAKMT_DEBUG_LEVEL_ERR -1
|
|
#define HSAKMT_DEBUG_LEVEL_DEFAULT 3
|
|
#define HSAKMT_DEBUG_LEVEL_WARNING 4
|
|
#define HSAKMT_DEBUG_LEVEL_INFO 6
|
|
#define HSAKMT_DEBUG_LEVEL_DEBUG 7
|
|
|
|
struct hsakmtRuntime {
|
|
hsakmtRuntime()
|
|
: dxg_fd(-1),
|
|
parent_pid(getpid()),
|
|
is_forked(false),
|
|
hsakmt_debug_level(HSAKMT_DEBUG_LEVEL_DEFAULT),
|
|
dxg_open_count(0),
|
|
hsakmt_mutex(PTHREAD_MUTEX_INITIALIZER),
|
|
hsakmt_is_dgpu(false),
|
|
is_svm_api_supported(false),
|
|
zfb_support(0),
|
|
vendor_packet_process(0),
|
|
check_avail_sysram(false),
|
|
max_single_alloc_size(0),
|
|
enable_thunk_sub_allocator(0),
|
|
local_heap_space_start_(0),
|
|
local_heap_space_size_(0),
|
|
system_heap_space_start_(0),
|
|
system_heap_space_size_(0),
|
|
handle_aperture_start_(0),
|
|
handle_aperture_size_(0),
|
|
default_node(1) {}
|
|
|
|
void HeapInit();
|
|
void HeapFini();
|
|
bool ReserveSvmSpace(uint64_t &base, uint64_t &size, uint64_t align);
|
|
bool FreeSvmSpace(uint64_t &base, uint64_t &size);
|
|
bool ReserveLocalHeapSpace();
|
|
bool FreeLocalHeapSpace();
|
|
void InitLocalHeapMgr();
|
|
bool ReserveSystemHeapSpace();
|
|
uint64_t SystemHeapSize() { return system_heap_space_size_; }
|
|
bool FreeSystemHeapSpace();
|
|
void InitSystemHeapMgr();
|
|
bool InitHandleApertureSpace();
|
|
void InitHandleApertureMgr();
|
|
ErrorCode HandleApertureAlloc(gpusize size, gpusize *out_gpu_virt_addr);
|
|
void HandleApertureFree(gpusize gpu_addr);
|
|
|
|
pthread_mutex_t hsakmt_mutex;
|
|
const char *dxg_device_name = "/dev/dxg";
|
|
long page_size;
|
|
int page_shift;
|
|
int dxg_fd = -1;
|
|
pid_t parent_pid = -1;
|
|
bool is_forked = false;
|
|
int hsakmt_debug_level = HSAKMT_DEBUG_LEVEL_DEFAULT;
|
|
unsigned long dxg_open_count;
|
|
bool hsakmt_is_dgpu;
|
|
bool is_svm_api_supported;
|
|
int zfb_support;
|
|
int vendor_packet_process;
|
|
bool check_avail_sysram;
|
|
size_t max_single_alloc_size;
|
|
int enable_thunk_sub_allocator;
|
|
uint32_t default_node;
|
|
|
|
/* local heap means bo's backend is vram of all GPUs */
|
|
uint64_t local_heap_space_start_;
|
|
uint64_t local_heap_space_size_;
|
|
|
|
/* manage the reserved local heap space which shared by CPU and GPUs */
|
|
std::unique_ptr<wsl::thunk::VaMgr> local_heap_mgr_;
|
|
|
|
/* system heap means bo's backend is system ram */
|
|
uint64_t system_heap_space_start_;
|
|
uint64_t system_heap_space_size_;
|
|
|
|
/* manage the reserved system heap space which shared by CPU and GPUs */
|
|
std::unique_ptr<wsl::thunk::VaMgr> system_heap_mgr_;
|
|
|
|
uint64_t handle_aperture_start_;
|
|
uint64_t handle_aperture_size_;
|
|
std::unique_ptr<wsl::thunk::VaMgr> handle_aperture_mgr_;
|
|
};
|
|
|
|
extern hsakmtRuntime *dxg_runtime;
|
|
|
|
#undef HSAKMTAPI
|
|
#define HSAKMTAPI __attribute__((visibility ("default")))
|
|
|
|
#if defined(__clang__)
|
|
#if __has_feature(address_sanitizer)
|
|
#define SANITIZER_AMDGPU 1
|
|
#endif
|
|
#endif
|
|
|
|
/*Avoid pointer-to-int-cast warning*/
|
|
#define PORT_VPTR_TO_UINT64(vptr) ((uint64_t)(unsigned long)(vptr))
|
|
|
|
/*Avoid int-to-pointer-cast warning*/
|
|
#define PORT_UINT64_TO_VPTR(v) ((void*)(unsigned long)(v))
|
|
|
|
#define CHECK_DXG_OPEN() \
|
|
do { if (dxg_runtime->dxg_open_count == 0 || dxg_runtime->is_forked) return HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED; } while (0)
|
|
|
|
extern int PAGE_SHIFT;
|
|
|
|
/* 64KB BigK fragment size for TLB efficiency */
|
|
#define GPU_BIGK_PAGE_SIZE (1 << 16)
|
|
|
|
/* 2MB huge page size for 4-level page tables on Vega10 and later GPUs */
|
|
#define GPU_HUGE_PAGE_SIZE (2 << 20)
|
|
|
|
#define CHECK_PAGE_MULTIPLE(x) \
|
|
do { if ((uint64_t)PORT_VPTR_TO_UINT64(x) % dxg_runtime->page_size) return HSAKMT_STATUS_INVALID_PARAMETER; } while(0)
|
|
|
|
#define ALIGN_UP(x,align) (((uint64_t)(x) + (align) - 1) & ~(uint64_t)((align)-1))
|
|
#define ALIGN_UP_32(x,align) (((uint32_t)(x) + (align) - 1) & ~(uint32_t)((align)-1))
|
|
#define PAGE_ALIGN_UP(x) ALIGN_UP(x,dxg_runtime->page_size)
|
|
#define BITMASK(n) ((n) ? (UINT64_MAX >> (sizeof(UINT64_MAX) * CHAR_BIT - (n))) : 0)
|
|
#define ARRAY_LEN(array) (sizeof(array) / sizeof(array[0]))
|
|
|
|
/* HSA Thunk logging usage */
|
|
#define get_thread_id() \
|
|
([]() -> std::string { \
|
|
std::stringstream str_thrd_id; \
|
|
str_thrd_id << std::hex << std::this_thread::get_id(); \
|
|
return str_thrd_id.str(); \
|
|
})()
|
|
#define hsakmt_print_common(stream, fmt, ...) \
|
|
do { \
|
|
fprintf(stream, "pid:%d tid:0x%s [%s] " fmt, getpid(), get_thread_id().c_str(), __FUNCTION__, ##__VA_ARGS__); \
|
|
fflush(stream); \
|
|
} while (false)
|
|
#ifdef NDEBUG
|
|
#define hsakmt_print(level, fmt, ...) \
|
|
do { } while (false)
|
|
#else
|
|
#define hsakmt_print(level, fmt, ...) \
|
|
do { \
|
|
if (level <= dxg_runtime->hsakmt_debug_level) { \
|
|
hsakmt_print_common(stdout, fmt, ##__VA_ARGS__); \
|
|
} \
|
|
} while (false)
|
|
#endif
|
|
|
|
#define pr_err(fmt, ...) \
|
|
hsakmt_print_common(stderr, fmt, ##__VA_ARGS__)
|
|
#define pr_warn(fmt, ...) \
|
|
hsakmt_print(HSAKMT_DEBUG_LEVEL_WARNING, fmt, ##__VA_ARGS__)
|
|
#define pr_info(fmt, ...) \
|
|
hsakmt_print(HSAKMT_DEBUG_LEVEL_INFO, fmt, ##__VA_ARGS__)
|
|
#define pr_debug(fmt, ...) \
|
|
hsakmt_print(HSAKMT_DEBUG_LEVEL_DEBUG, fmt, ##__VA_ARGS__)
|
|
#define pr_err_once(fmt, ...) \
|
|
({ \
|
|
static bool __print_once; \
|
|
if (!__print_once) { \
|
|
__print_once = true; \
|
|
pr_err(fmt, ##__VA_ARGS__); \
|
|
} \
|
|
})
|
|
#define pr_warn_once(fmt, ...) \
|
|
({ \
|
|
static bool __print_once; \
|
|
if (!__print_once) { \
|
|
__print_once = true; \
|
|
pr_warn(fmt, ##__VA_ARGS__); \
|
|
} \
|
|
})
|
|
|
|
/* Expects HSA_ENGINE_ID.ui32, returns gfxv (full) in hex */
|
|
#define HSA_GET_GFX_VERSION_FULL(ui32) \
|
|
(((ui32.Major) << 16) | ((ui32.Minor) << 8) | (ui32.Stepping))
|
|
|
|
HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id);
|
|
HSAKMT_STATUS gpuid_to_nodeid(uint32_t gpu_id, uint32_t* node_id);
|
|
bool prefer_ats(HSAuint32 node_id);
|
|
uint16_t get_device_id_by_node_id(HSAuint32 node_id);
|
|
uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id);
|
|
uint32_t get_direct_link_cpu(uint32_t gpu_node);
|
|
|
|
HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties& props);
|
|
HSAKMT_STATUS topology_get_node_props(HSAuint32 NodeId,
|
|
HsaNodeProperties *NodeProperties);
|
|
HSAKMT_STATUS topology_get_iolink_props(HSAuint32 NodeId,
|
|
HSAuint32 NumIoLinks,
|
|
HsaIoLinkProperties *IoLinkProperties);
|
|
void topology_setup_is_dgpu_param(HsaNodeProperties *props);
|
|
|
|
HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags);
|
|
|
|
uint32_t get_num_sysfs_nodes(void);
|
|
|
|
bool is_forked_child(void);
|
|
|
|
void clear_allocation_map(void);
|
|
|
|
class BlockAllocator {
|
|
private:
|
|
static const size_t block_size_ = 128 * 1024 * 1024; // 128MB blocks.
|
|
|
|
public:
|
|
void* alloc(size_t request_size, size_t& allocated_size) const;
|
|
void free(void* ptr, size_t length) const;
|
|
size_t block_size() const { return block_size_; }
|
|
};
|
|
|
|
void reset_suballocator(void);
|
|
void trim_suballocator(void);
|
|
|
|
HSAKMT_STATUS hsaKmtAllocMemoryAlignInternal(HSAuint32 PreferredNode,
|
|
HSAuint64 SizeInBytes,
|
|
HSAuint64 Alignment,
|
|
HsaMemFlags MemFlags,
|
|
void **MemoryAddress,
|
|
bool SkipSubAlloc = false);
|
|
|
|
HSAKMT_STATUS hsaKmtFreeMemoryInternal(void *MemoryAddress,
|
|
HSAuint64 SizeInBytes,
|
|
bool SkipSubAlloc = false);
|
|
|
|
bool queue_acquire_buffer(void *MemoryAddress);
|
|
bool queue_release_buffer(void *MemoryAddress);
|
|
/* Calculate VGPR and SGPR register file size per CU */
|
|
uint32_t get_vgpr_size_per_cu(HSA_ENGINE_ID id);
|
|
#define SGPR_SIZE_PER_CU 0x4000
|
|
|
|
bool is_ipc_sysmemfd(int fd);
|
|
|
|
HSAKMT_STATUS import_dmabuf_fd(int DMABufFd,
|
|
uint32_t NodeId,
|
|
bool alloc_va,
|
|
bool is_ipc_memfd,
|
|
wsl::thunk::GpuMemoryHandle *GpuMemHandle);
|
|
|
|
#endif
|