//////////////////////////////////////////////////////////////////////////////// // // The University of Illinois/NCSA // Open Source License (NCSA) // // Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved. // // Developed by: // // AMD Research and AMD HSA Software Development // // Advanced Micro Devices, Inc. // // www.amd.com // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to // deal with the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // - Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimers. // - Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimers in // the documentation and/or other materials provided with the distribution. // - Neither the names of Advanced Micro Devices, Inc, // nor the names of its contributors may be used to endorse or promote // products derived from this Software without specific prior written // permission. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL // THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR // OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, // ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS WITH THE SOFTWARE. // //////////////////////////////////////////////////////////////////////////////// // HSA runtime C++ interface file. #ifndef HSA_RUNTME_CORE_INC_RUNTIME_H_ #define HSA_RUNTME_CORE_INC_RUNTIME_H_ #include #include #include #include #include #include #include #if defined(__linux__) #include #include #endif #include "core/inc/hsa_ext_interface.h" #include "core/inc/hsa_internal.h" #include "core/inc/hsa_ext_amd_impl.h" #include "core/inc/agent.h" #include "core/inc/exceptions.h" #include "core/inc/memory_region.h" #include "core/inc/signal.h" #include "core/inc/interrupt_signal.h" #include "core/inc/svm_profiler.h" #include "core/util/flag.h" #include "core/util/locks.h" #include "core/util/os.h" #include "core/util/utils.h" #include "core/inc/amd_loader_context.hpp" #include "core/inc/amd_hsa_code.hpp" #if defined(__clang__) #if __has_feature(address_sanitizer) #define SANITIZER_AMDGPU 1 #endif #endif //---------------------------------------------------------------------------// // Constants // //---------------------------------------------------------------------------// #define HSA_ARGUMENT_ALIGN_BYTES 16 #define HSA_QUEUE_ALIGN_BYTES 64 #define HSA_PACKET_ALIGN_BYTES 64 //Avoids include namespace rocr { namespace AMD { class MemoryRegion; } // namespace amd namespace core { extern bool g_use_interrupt_wait; extern bool g_use_mwaitx; /// @brief Runtime class provides the following functions: /// - open and close connection to kernel driver. /// - load supported extension library (image and finalizer). /// - load tools library. /// - expose supported agents. /// - allocate and free memory. /// - memory copy and fill. /// - grant access to memory (dgpu memory pool extension). /// - maintain loader state. /// - monitor asynchronous event from agent. class Runtime { friend class AMD::MemoryRegion; public: /// @brief Structure to describe connectivity between agents. struct LinkInfo { LinkInfo() : num_hop(0), info{0} {} uint32_t num_hop; hsa_amd_memory_pool_link_info_t info; }; struct KfdVersion_t { HsaVersionInfo version; bool supports_exception_debugging; bool supports_event_age; bool supports_core_dump; }; /// @brief Open connection to kernel driver and increment reference count. static hsa_status_t Acquire(); /// @brief Decrement reference count and close connection to kernel driver. static hsa_status_t Release(); /// @brief Checks if connection to kernel driver is opened. /// @retval True if the connection to kernel driver is opened. static bool IsOpen(); // @brief Callback handler for HW Exceptions. static bool HwExceptionHandler(hsa_signal_value_t val, void* arg); // @brief Callback handler for VM fault access. static bool VMFaultHandler(hsa_signal_value_t val, void* arg); // @brief Print known allocations near ptr. static void PrintMemoryMapNear(void* ptr); /// @brief Singleton object of the runtime. static Runtime* runtime_singleton_; /// @brief Insert agent into agent list ::agents_. /// @param [in] agent Pointer to the agent object. void RegisterAgent(Agent* agent, bool Enabled); /// @brief Delete all agent objects from ::agents_. void DestroyAgents(); /// @brief Set the number of links connecting the agents in the platform. void SetLinkCount(size_t num_link); /// @brief Register link information connecting @p node_id_from and @p /// node_id_to. /// @param [in] node_id_from Node id of the source node. /// @param [in] node_id_to Node id of the destination node. /// @param [in] link_info The link information between source and destination /// nodes. void RegisterLinkInfo(uint32_t node_id_from, uint32_t node_id_to, uint32_t num_hop, hsa_amd_memory_pool_link_info_t& link_info); /// @brief Query link information between two nodes. /// @param [in] node_id_from Node id of the source node. /// @param [in] node_id_to Node id of the destination node. /// @retval The link information between source and destination nodes. const LinkInfo GetLinkInfo(uint32_t node_id_from, uint32_t node_id_to); /// @brief Invoke the user provided call back for each agent in the agent /// list. /// /// @param [in] callback User provided callback function. /// @param [in] data User provided pointer as input for @p callback. /// /// @retval ::HSA_STATUS_SUCCESS if the callback function for each traversed /// agent returns ::HSA_STATUS_SUCCESS. hsa_status_t IterateAgent(hsa_status_t (*callback)(hsa_agent_t agent, void* data), void* data); /// @brief Allocate memory on a particular region. /// /// @param [in] region Pointer to region object. /// @param [in] size Allocation size in bytes. /// @param [in] alloc_flags Modifiers to pass to MemoryRegion allocator. /// @param [out] address Pointer to store the allocation result. /// /// @retval ::HSA_STATUS_SUCCESS If allocation is successful. hsa_status_t AllocateMemory(const MemoryRegion* region, size_t size, MemoryRegion::AllocateFlags alloc_flags, void** address, int agent_node_id = 0); /// @brief Free memory previously allocated with AllocateMemory. /// /// @param [in] ptr Address of the memory to be freed. /// /// @retval ::HSA_STATUS_ERROR If @p ptr is not the address of previous /// allocation via ::core::Runtime::AllocateMemory /// @retval ::HSA_STATUS_SUCCESS if @p ptr is successfully released. hsa_status_t FreeMemory(void* ptr); hsa_status_t RegisterReleaseNotifier(void* ptr, hsa_amd_deallocation_callback_t callback, void* user_data); hsa_status_t DeregisterReleaseNotifier(void* ptr, hsa_amd_deallocation_callback_t callback); /// @brief Blocking memory copy from src to dst. /// /// @param [in] dst Memory address of the destination. /// @param [in] src Memory address of the source. /// @param [in] size Copy size in bytes. /// /// @retval ::HSA_STATUS_SUCCESS if memory copy is successful and completed. hsa_status_t CopyMemory(void* dst, const void* src, size_t size); /// @brief Non-blocking memory copy from src to dst. /// /// @details The memory copy will be performed after all signals in /// @p dep_signals have value of 0. On completion @p completion_signal /// will be decremented. /// /// @param [in] dst Memory address of the destination. /// @param [in] dst_agent Agent object associated with the destination. This /// agent should be able to access the destination and source. /// @param [in] src Memory address of the source. /// @param [in] src_agent Agent object associated with the source. This /// agent should be able to access the destination and source. /// @param [in] size Copy size in bytes. /// @param [in] dep_signals Array of signal dependency. /// @param [in] completion_signal Completion signal object. /// /// @retval ::HSA_STATUS_SUCCESS if copy command has been submitted /// successfully to the agent DMA queue. hsa_status_t CopyMemory(void* dst, core::Agent* dst_agent, const void* src, core::Agent* src_agent, size_t size, std::vector& dep_signals, core::Signal& completion_signal); /// @brief Non-blocking memory copy from src to dst on engine_id. /// /// @details All semantics and params are dentical to CopyMemory /// with the exception of engine_id. /// /// @param [in] engine_id Target engine to copy on. /// /// @param [in] force_copy_on_sdma By default, a blit kernel copy is used /// when dst_agent == src_agent. Setting this to true will force the copy /// over SDMA1. /// /// @retval ::HSA_STATUS_SUCCESS if copy command has been submitted /// successfully to the agent DMA queue. hsa_status_t CopyMemoryOnEngine(void* dst, core::Agent* dst_agent, const void* src, core::Agent* src_agent, size_t size, std::vector& dep_signals, core::Signal& completion_signal, hsa_amd_sdma_engine_id_t engine_id, bool force_copy_on_sdma); /// @brief Return SDMA availability status for copy direction /// /// @param [in] dst_agent Destination agent. /// @param [in] src_agent Source agent. /// @param [out] engine_ids_mask Mask of engine_ids. /// /// @retval HSA_STATUS_SUCCESS DMA engines are available /// @retval HSA_STATUS_ERROR_OUT_OF_RESOURCES DMA engines are not available hsa_status_t CopyMemoryStatus(core::Agent* dst_agent, core::Agent* src_agent, uint32_t *engine_ids_mask); /// @brief Fill the first @p count of uint32_t in ptr with value. /// /// @param [in] ptr Memory address to be filled. /// @param [in] value The value/pattern that will be used to set @p ptr. /// @param [in] count Number of uint32_t element to be set. /// /// @retval ::HSA_STATUS_SUCCESS if memory fill is successful and completed. hsa_status_t FillMemory(void* ptr, uint32_t value, size_t count); /// @brief Set agents as the whitelist to access ptr. /// /// @param [in] num_agents The number of agent handles in @p agents array. /// @param [in] agents Agent handle array. /// @param [in] ptr Pointer of memory previously allocated via /// core::Runtime::AllocateMemory. /// /// @retval ::HSA_STATUS_SUCCESS The whitelist has been configured /// successfully and all agents in the @p agents could start accessing @p ptr. hsa_status_t AllowAccess(uint32_t num_agents, const hsa_agent_t* agents, const void* ptr); /// @brief Query system information. /// /// @param [in] attribute System info attribute to query. /// @param [out] value Pointer to store the attribute value. /// /// @retval HSA_STATUS_SUCCESS The attribute is valid and the @p value is /// set. hsa_status_t GetSystemInfo(hsa_system_info_t attribute, void* value); /// @brief Register a callback function @p handler that is associated with /// @p signal to asynchronous event monitor thread. /// /// @param [in] signal Signal handle associated with @p handler. /// @param [in] cond The condition to execute the @p handler. /// @param [in] value The value to compare with @p signal value. If the /// comparison satisfy @p cond, the @p handler will be called. /// @param [in] arg Pointer to the argument that will be provided to @p /// handler. /// /// @retval ::HSA_STATUS_SUCCESS Registration is successful. hsa_status_t SetAsyncSignalHandler(hsa_signal_t signal, hsa_signal_condition_t cond, hsa_signal_value_t value, hsa_amd_signal_handler handler, void* arg); hsa_status_t InteropMap(uint32_t num_agents, Agent** agents, int interop_handle, uint32_t flags, size_t* size, void** ptr, size_t* metadata_size, const void** metadata); hsa_status_t InteropUnmap(void* ptr); struct PtrInfoBlockData { void* base; size_t length; core::Agent* agentOwner; }; hsa_status_t PtrInfo(const void* ptr, hsa_amd_pointer_info_t* info, void* (*alloc)(size_t), uint32_t* num_agents_accessible, hsa_agent_t** accessible, PtrInfoBlockData* block_info = nullptr); hsa_status_t SetPtrInfoData(const void* ptr, void* userptr); hsa_status_t IPCCreate(void* ptr, size_t len, hsa_amd_ipc_memory_t* handle); hsa_status_t IPCAttach(const hsa_amd_ipc_memory_t* handle, size_t len, uint32_t num_agents, Agent** mapping_agents, void** mapped_ptr); hsa_status_t IPCDetach(void* ptr); hsa_status_t SetSvmAttrib(void* ptr, size_t size, hsa_amd_svm_attribute_pair_t* attribute_list, size_t attribute_count); hsa_status_t GetSvmAttrib(void* ptr, size_t size, hsa_amd_svm_attribute_pair_t* attribute_list, size_t attribute_count); hsa_status_t SvmPrefetch(void* ptr, size_t size, hsa_agent_t agent, uint32_t num_dep_signals, const hsa_signal_t* dep_signals, hsa_signal_t completion_signal); hsa_status_t DmaBufExport(const void* ptr, size_t size, int* dmabuf, uint64_t* offset); hsa_status_t DmaBufClose(int dmabuf); hsa_status_t VMemoryAddressReserve(void** ptr, size_t size, uint64_t address, uint64_t flags); hsa_status_t VMemoryAddressFree(void* ptr, size_t size); hsa_status_t VMemoryHandleCreate(const MemoryRegion* region, size_t size, MemoryRegion::AllocateFlags alloc_flags, uint64_t flags, hsa_amd_vmem_alloc_handle_t* memoryHandle); hsa_status_t VMemoryHandleRelease(hsa_amd_vmem_alloc_handle_t memoryHandle); hsa_status_t VMemoryHandleMap(void* va, size_t size, size_t in_offset, hsa_amd_vmem_alloc_handle_t memoryHandle, uint64_t flags); hsa_status_t VMemoryHandleUnmap(void* va, size_t size); hsa_status_t VMemorySetAccess(void* va, size_t size, const hsa_amd_memory_access_desc_t* desc, size_t desc_cnt); hsa_status_t VMemoryGetAccess(const void* va, hsa_access_permission_t* perms, hsa_agent_t agent_handle); hsa_status_t VMemoryExportShareableHandle(int* dmabuf_fd, const hsa_amd_vmem_alloc_handle_t handle, const uint64_t flags); hsa_status_t VMemoryImportShareableHandle(const int dmabuf_fd, hsa_amd_vmem_alloc_handle_t* handle); hsa_status_t VMemoryRetainAllocHandle(hsa_amd_vmem_alloc_handle_t* memoryHandle, void* addr); hsa_status_t VMemoryGetAllocPropertiesFromHandle(const hsa_amd_vmem_alloc_handle_t memoryHandle, const core::MemoryRegion** mem_region, hsa_amd_memory_type_t* type); const std::vector& cpu_agents() { return cpu_agents_; } const std::vector& gpu_agents() { return gpu_agents_; } const std::vector& disabled_gpu_agents() { return disabled_gpu_agents_; } const std::vector& gpu_ids() { return gpu_ids_; } Agent* agent_by_gpuid(uint32_t gpuid) { return agents_by_gpuid_[gpuid]; } Agent* region_gpu() { return region_gpu_; } const std::vector& system_regions_fine() const { return system_regions_fine_; } const std::vector& system_regions_coarse() const { return system_regions_coarse_; } amd::hsa::loader::Loader* loader() { return loader_; } amd::LoaderContext* loader_context() { return &loader_context_; } amd::hsa::code::AmdHsaCodeManager* code_manager() { return &code_manager_; } std::function& system_allocator() { return system_allocator_; } std::function& system_deallocator() { return system_deallocator_; } const Flag& flag() const { return flag_; } ExtensionEntryPoints extensions_; hsa_status_t SetCustomSystemEventHandler(hsa_amd_system_event_callback_t callback, void* data); hsa_status_t SetInternalQueueCreateNotifier(hsa_amd_runtime_queue_notifier callback, void* user_data); void InternalQueueCreateNotify(const hsa_queue_t* queue, hsa_agent_t agent); SharedSignalPool_t* GetSharedSignalPool() { return &SharedSignalPool; } InterruptSignal::EventPool* GetEventPool() { return &EventPool; } uint64_t sys_clock_freq() const { return sys_clock_freq_; } void KfdVersion(const HsaVersionInfo& version) { kfd_version.version = version; if (version.KernelInterfaceMajorVersion == 1 && version.KernelInterfaceMinorVersion >= 14) kfd_version.supports_event_age = true; } void KfdVersion(bool exception_debugging, bool core_dump) { kfd_version.supports_exception_debugging = exception_debugging; kfd_version.supports_core_dump = core_dump; } KfdVersion_t KfdVersion() const { return kfd_version; } bool VirtualMemApiSupported() const { return virtual_mem_api_supported_; } bool XnackEnabled() const { return xnack_enabled_; } void XnackEnabled(bool enable) { xnack_enabled_ = enable; } protected: static void AsyncEventsLoop(void*); static void AsyncIPCSockServerConnLoop(void*); struct AllocationRegion { AllocationRegion() : region(NULL), size(0), size_requested(0), alloc_flags(core::MemoryRegion::AllocateNoFlags), user_ptr(nullptr), ldrm_bo(NULL) {} AllocationRegion(const MemoryRegion* region_arg, size_t size_arg, size_t size_requested, MemoryRegion::AllocateFlags alloc_flags) : region(region_arg), size(size_arg), size_requested(size_requested), alloc_flags(alloc_flags), user_ptr(nullptr) {} struct notifier_t { void* ptr; AMD::callback_t callback; void* user_data; }; const MemoryRegion* region; size_t size; /* actual size = align_up(size_requested, granularity) */ size_t size_requested; /* size requested by user */ MemoryRegion::AllocateFlags alloc_flags; void* user_ptr; std::unique_ptr> notifiers; amdgpu_bo_handle ldrm_bo; }; struct AsyncEventsControl { AsyncEventsControl() : async_events_thread_(NULL) {} void Shutdown(); hsa_signal_t wake; os::Thread async_events_thread_; HybridMutex lock; bool exit; }; struct AsyncEvents { void PushBack(hsa_signal_t signal, hsa_signal_condition_t cond, hsa_signal_value_t value, hsa_amd_signal_handler handler, void* arg); void CopyIndex(size_t dst, size_t src); size_t Size(); void PopBack(); void Clear(); std::vector signal_; std::vector cond_; std::vector value_; std::vector handler_; std::vector arg_; }; struct PrefetchRange; typedef std::map prefetch_map_t; struct PrefetchOp { void* base; size_t size; uint32_t node_id; int remaining_deps; hsa_signal_t completion; std::vector dep_signals; prefetch_map_t::iterator prefetch_map_entry; }; struct PrefetchRange { PrefetchRange() {} PrefetchRange(size_t Bytes, PrefetchOp* Op) : bytes(Bytes), op(Op) {} size_t bytes; PrefetchOp* op; prefetch_map_t::iterator prev; prefetch_map_t::iterator next; }; // Will be created before any user could call hsa_init but also could be // destroyed before incorrectly written programs call hsa_shutdown. static KernelMutex bootstrap_lock_; Runtime(); Runtime(const Runtime&); Runtime& operator=(const Runtime&); ~Runtime() {} /// @brief Open connection to kernel driver. hsa_status_t Load(); /// @brief Close connection to kernel driver and cleanup resources. void Unload(); /// @brief Dynamically load extension libraries (images, finalizer) and /// call OnLoad method on each loaded library. void LoadExtensions(); /// @brief Call OnUnload method on each extension library then close it. void UnloadExtensions(); /// @brief Dynamically load tool libraries and call OnUnload method on each /// loaded library. void LoadTools(); /// @brief Call OnUnload method of each tool library. void UnloadTools(); /// @brief Close tool libraries. void CloseTools(); // @brief Binds Error handlers to this node. void BindErrorHandlers(); // @brief Acquire snapshot of system event handlers. // Returns a copy to avoid holding a lock during callbacks. std::vector, void*>> GetSystemEventHandlers(); /// @brief Get the index of ::link_matrix_. /// @param [in] node_id_from Node id of the source node. /// @param [in] node_id_to Node id of the destination node. /// @retval Index in ::link_matrix_. uint32_t GetIndexLinkInfo(uint32_t node_id_from, uint32_t node_id_to); /// @brief Get most recently issued SVM prefetch agent for the range in question. Agent* GetSVMPrefetchAgent(void* ptr, size_t size); /// @brief Get the highest used node id. uint32_t max_node_id() const { return agents_by_node_.rbegin()->first; } // Mutex object to protect multithreaded access to ::allocation_map_. // Also ensures atomicity of pointer info queries by interlocking // KFD map/unmap, register/unregister, and access to hsaKmtQueryPointerInfo // registered & mapped arrays. KernelSharedMutex memory_lock_; // Array containing tools library handles. std::vector tool_libs_; // Agent list containing all CPU agents in the platform. std::vector cpu_agents_; // Agent list containing all compatible GPU agents in the platform. std::vector gpu_agents_; // Agent list containing incompletely initialized GPU agents not to be used by the process. std::vector disabled_gpu_agents_; // Agent map containing all agents indexed by their KFD node IDs. std::map > agents_by_node_; // Agent map containing all agents indexed by their KFD gpuid. std::map agents_by_gpuid_; // Agent list containing all compatible gpu agent ids in the platform. std::vector gpu_ids_; // List of all fine grain system memory region in the platform. std::vector system_regions_fine_; // List of all coarse grain system memory region in the platform. std::vector system_regions_coarse_; // Matrix of IO link. std::vector link_matrix_; // Loader instance. amd::hsa::loader::Loader* loader_; // Loader context. amd::LoaderContext loader_context_; // Code object manager. amd::hsa::code::AmdHsaCodeManager code_manager_; // Contains the region, address, and size of previously allocated memory. std::map allocation_map_; // Pending prefetch containers. KernelMutex prefetch_lock_; prefetch_map_t prefetch_map_; // Allocator using ::system_region_ std::function system_allocator_; // Deallocator using ::system_region_ std::function system_deallocator_; // Deprecated HSA Region API GPU (for legacy APU support only) Agent* region_gpu_; AsyncEventsControl async_events_control_; AsyncEvents async_events_; AsyncEvents new_async_events_; // System clock frequency. uint64_t sys_clock_freq_; // Number of Numa Nodes size_t num_nodes_; // @brief AMD HSA event to monitor for virtual memory access fault. HsaEvent* vm_fault_event_; // @brief HSA signal to contain the VM fault event. Signal* vm_fault_signal_; // @brief AMD HSA event to monitor for HW exceptions. HsaEvent* hw_exception_event_; // @brief HSA signal to contain the HW exceptionevent. Signal* hw_exception_signal_; // Custom system event handlers. std::vector, void*>> system_event_handlers_; // System event handler lock KernelMutex system_event_lock_; // Internal queue creation notifier AMD::callback_t internal_queue_create_notifier_; void* internal_queue_create_notifier_user_data_; // Holds reference count to runtime object. std::atomic ref_count_; // Track environment variables. Flag flag_; // Pools memory for SharedSignal (Signal ABI blocks) SharedSignalPool_t SharedSignalPool; // Pools KFD Events for InterruptSignal InterruptSignal::EventPool EventPool; // Kfd version KfdVersion_t kfd_version; std::unique_ptr svm_profile_; // IPC DMA buf unix domain socket server dmabuf FD passing int ipc_sock_server_fd_; std::map> ipc_sock_server_conns_; KernelMutex ipc_sock_server_lock_; private: void CheckVirtualMemApiSupport(); int GetAmdgpuDeviceArgs(Agent* agent, amdgpu_bo_handle bo, int* drm_fd, uint64_t* cpu_addr); bool virtual_mem_api_supported_; bool xnack_enabled_; typedef void* ThunkHandle; struct AddressHandle { AddressHandle() : size(0), use_count(0) {} AddressHandle(size_t size) : size(size), use_count(0) {} size_t size; int use_count; }; std::map reserved_address_map_; // Indexed by VA struct MemoryHandle { MemoryHandle() : region(NULL), size(0), ref_count(0), thunk_handle(NULL), alloc_flag(0) {} MemoryHandle(const MemoryRegion* region, size_t size, uint64_t flags_unused, ThunkHandle thunk_handle, MemoryRegion::AllocateFlags alloc_flag) : region(region), size(size), ref_count(1), use_count(0), thunk_handle(thunk_handle), alloc_flag(alloc_flag) {} static __forceinline hsa_amd_vmem_alloc_handle_t Convert(void* handle) { hsa_amd_vmem_alloc_handle_t ret_handle = { static_cast(reinterpret_cast(handle))}; return ret_handle; } __forceinline core::Agent* agentOwner() const { return region->owner(); } const MemoryRegion* region; size_t size; int ref_count; int use_count; ThunkHandle thunk_handle; // handle returned by hsaKmtAllocMemory(NoAddress = 1) MemoryRegion::AllocateFlags alloc_flag; }; std::map memory_handle_map_; struct MappedHandle; struct MappedHandleAllowedAgent { MappedHandleAllowedAgent() : va(NULL), permissions(HSA_ACCESS_PERMISSION_NONE), mappedHandle(NULL), ldrm_bo(0) {} MappedHandleAllowedAgent(MappedHandle* _mappedHandle, Agent* targetAgent, void* va, size_t size, hsa_access_permission_t perms); ~MappedHandleAllowedAgent(); hsa_status_t RemoveAccess(); hsa_status_t EnableAccess(hsa_access_permission_t perms); void* va; size_t size; Agent* targetAgent; hsa_access_permission_t permissions; MappedHandle* mappedHandle; amdgpu_bo_handle ldrm_bo; }; struct MappedHandle { MappedHandle() : mem_handle(NULL), address_handle(NULL), offset(0), mmap_offset(0), size(0), drm_fd(-1), drm_cpu_addr(NULL), ldrm_bo(0) {} MappedHandle(MemoryHandle* mem_handle, AddressHandle* address_handle, uint64_t offset, size_t size, int drm_fd, void* drm_cpu_addr, hsa_access_permission_t perm, amdgpu_bo_handle bo) : mem_handle(mem_handle), address_handle(address_handle), offset(offset), mmap_offset(0), size(size), drm_fd(drm_fd), drm_cpu_addr(drm_cpu_addr), ldrm_bo(bo) {} __forceinline core::Agent* agentOwner() const { return mem_handle->region->owner(); } MemoryHandle* mem_handle; AddressHandle* address_handle; uint64_t offset; uint64_t mmap_offset; size_t size; int drm_fd; void* drm_cpu_addr; // CPU Buffer address amdgpu_bo_handle ldrm_bo; std::map allowed_agents; }; std::map mapped_handle_map_; // Indexed by VA // Frees runtime memory when the runtime library is unloaded if safe to do so. // Failure to release the runtime indicates an incorrect application but is // common (example: calls library routines at process exit). friend class RuntimeCleanup; void InitIPCDmaBufSupport(); bool ipc_dmabuf_supported_; }; } // namespace core } // namespace rocr #endif // header guard