From e16de03a2ef3407c2cc922c85dc74e966a66a738 Mon Sep 17 00:00:00 2001 From: foreman Date: Mon, 13 May 2019 12:19:10 -0400 Subject: [PATCH] P4 to Git Change 1781985 by gandryey@gera-win10 on 2019/05/13 11:54:40 SWDEV-189140 - Add P2P support in PAL path - PAL requires P2P resource open on the usage device. Add the new interface to open the resource - Add a hidden P2P device object creation into amd::Memory. It can be activated with OCL context that has a single device. Affected files ... ... //depot/stg/opencl/drivers/opencl/api/opencl/amdocl/cl_p2p_amd.cpp#2 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/device.hpp#337 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/paldevice.cpp#134 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palmemory.cpp#25 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palresource.cpp#74 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palresource.hpp#28 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palsettings.cpp#80 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palsettings.hpp#23 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palvirtual.cpp#133 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocdevice.cpp#126 edit ... //depot/stg/opencl/drivers/opencl/runtime/platform/command.cpp#93 edit ... //depot/stg/opencl/drivers/opencl/runtime/platform/memory.cpp#136 edit ... //depot/stg/opencl/drivers/opencl/runtime/platform/memory.hpp#109 edit ... //depot/stg/opencl/drivers/opencl/runtime/utils/flags.hpp#306 edit --- rocclr/runtime/device/device.hpp | 102 ++++++++++++---------- rocclr/runtime/device/pal/paldevice.cpp | 34 +++++++- rocclr/runtime/device/pal/palmemory.cpp | 3 +- rocclr/runtime/device/pal/palresource.cpp | 98 +++++++++++++++------ rocclr/runtime/device/pal/palresource.hpp | 20 +++-- rocclr/runtime/device/pal/palsettings.cpp | 8 +- rocclr/runtime/device/pal/palsettings.hpp | 3 +- rocclr/runtime/device/pal/palvirtual.cpp | 36 ++++---- rocclr/runtime/device/rocm/rocdevice.cpp | 3 +- rocclr/runtime/platform/command.cpp | 9 +- rocclr/runtime/platform/memory.cpp | 56 ++++++++---- rocclr/runtime/platform/memory.hpp | 77 +++++++++------- rocclr/runtime/utils/flags.hpp | 14 +-- 13 files changed, 306 insertions(+), 157 deletions(-) diff --git a/rocclr/runtime/device/device.hpp b/rocclr/runtime/device/device.hpp index f45ab12c37..bec4a1669b 100644 --- a/rocclr/runtime/device/device.hpp +++ b/rocclr/runtime/device/device.hpp @@ -19,10 +19,10 @@ #include "devkernel.hpp" #include "amdocl/cl_profile_amd.h" -#if defined(WITH_LIGHTNING_COMPILER) && ! defined(USE_COMGR_LIBRARY) +#if defined(WITH_LIGHTNING_COMPILER) && !defined(USE_COMGR_LIBRARY) #include "caching/cache.hpp" #include "driver/AmdCompiler.h" -#endif // defined(WITH_LIGHTNING_COMPILER) && ! defined(USE_COMGR_LIBRARY) +#endif // defined(WITH_LIGHTNING_COMPILER) && ! defined(USE_COMGR_LIBRARY) #include "acl.h" #include "hwdebug.hpp" @@ -77,9 +77,9 @@ struct Coord3D; namespace option { class Options; -} // option +} // namespace option -} +} // namespace amd enum OclExtensions { ClKhrFp64 = 0, @@ -645,13 +645,13 @@ class Memory : public amd::HeapObject { uint mapFlags, //!< Map flags size_t* rowPitch = NULL, //!< Row pitch for the mapped memory size_t* slicePitch = NULL //!< Slice for the mapped memory - ) { + ) { return NULL; } virtual bool pinSystemMemory(void* hostPtr, //!< System memory address size_t size //!< Size of allocated system memory - ) { + ) { return true; } @@ -669,7 +669,7 @@ class Memory : public amd::HeapObject { uint numLayers = 0, //!< End layer for multilayer map size_t* rowPitch = NULL, //!< Row pitch for the device memory size_t* slicePitch = NULL //!< Slice pitch for the device memory - ) { + ) { amd::Image* image = owner()->asImage(); if (image != NULL) { *rowPitch = image->getRowPitch(); @@ -681,7 +681,7 @@ class Memory : public amd::HeapObject { //! Unmap the device memory virtual void cpuUnmap(VirtualDevice& vDev //!< Virtual device for unmap operaiton - ) {} + ) {} //! Saves map info for this object //! @note: It's not a thread safe operation, the app must implement @@ -692,7 +692,7 @@ class Memory : public amd::HeapObject { uint mapFlags, //!< Map flags bool entire, //!< True if the enitre memory was mapped amd::Image* baseMip = nullptr //!< The base mip level for map - ); + ); const WriteMapInfo* writeMapInfo(const void* mapAddress) const { // Unmap must be serialized. @@ -762,9 +762,10 @@ class Memory : public amd::HeapObject { //! NB, the map data below is for an API-level map (from clEnqueueMapBuffer), //! not a physical map. When a memory object does not use USE_HOST_PTR we //! can use a remote resource and DMA, avoiding the additional CPU memcpy. - amd::Memory* mapMemory_; //!< Memory used as map target buffer - volatile size_t indirectMapCount_; //!< Number of maps - std::unordered_map writeMapInfo_; //!< Saved write map info for partial unmap + amd::Memory* mapMemory_; //!< Memory used as map target buffer + volatile size_t indirectMapCount_; //!< Number of maps + std::unordered_map + writeMapInfo_; //!< Saved write map info for partial unmap //! Increment map count void incIndMapCount() { ++indirectMapCount_; } @@ -772,7 +773,7 @@ class Memory : public amd::HeapObject { //! Decrement map count virtual void decIndMapCount() {} - size_t size_; //!< Memory size + size_t size_; //!< Memory size private: //! Disable default copy constructor @@ -797,8 +798,8 @@ class Sampler : public amd::HeapObject { const address hwState() const { return hwState_; } protected: - uint64_t hwSrd_; //!< Device specific HW state for the sampler - address hwState_; //!< CPU pointer to HW state + uint64_t hwSrd_; //!< Device specific HW state for the sampler + address hwState_; //!< CPU pointer to HW state private: //! Disable default copy constructor @@ -873,11 +874,11 @@ class ClBinary : public amd::HeapObject { //! Store compile options into OCL binary file void storeCompileOptions(const std::string& compileOptions //!< the compile options to be stored - ); + ); //! Store link options into OCL binary file void storeLinkOptions(const std::string& linkOptions //!< the link options to be stored - ); + ); //! Check if the binary is recompilable bool isRecompilable(std::string& llvmBinary, amd::OclElf::oclElfPlatform thePlatform); @@ -967,11 +968,11 @@ class ClBinary : public amd::HeapObject { void release(); const char* binary_; //!< binary data - size_t size_; //!< binary size - uint flags_; //!< CL binary object flags + size_t size_; //!< binary size + uint flags_; //!< CL binary object flags const char* origBinary_; //!< original binary data - size_t origSize_; //!< original binary size + size_t origSize_; //!< original binary size int encryptCode_; //!< Encryption Code for input binary (0 for not encrypted) @@ -1110,21 +1111,22 @@ class MemObjMap : public AllStatic { public: static size_t size(); //!< obtain the size of the container static void AddMemObj(const void* k, - amd::Memory* v); //!< add the host mem pointer and buffer in the container + amd::Memory* v); //!< add the host mem pointer and buffer in the container static void RemoveMemObj(const void* k); //!< Remove an entry of mem object from the container static amd::Memory* FindMemObj( const void* k); //!< find the mem object based on the input pointer private: - static std::map MemObjMap_; //!< the mem object<->hostptr information container - static amd::Monitor AllocatedLock_; //!< amd monitor locker + static std::map + MemObjMap_; //!< the mem object<->hostptr information container + static amd::Monitor AllocatedLock_; //!< amd monitor locker }; /*! \addtogroup Runtime -* @{ -* -* \addtogroup Device Device Abstraction -* @{ -*/ + * @{ + * + * \addtogroup Device Device Abstraction + * @{ + */ class Device : public RuntimeObject { protected: typedef aclCompiler Compiler; @@ -1144,7 +1146,7 @@ class Device : public RuntimeObject { bool create(Device* device, //!< Device object const char* extraKernel = NULL, //!< Extra kernels from the device layer const char* extraOptions = NULL //!< Extra compilation options - ); + ); }; virtual Compiler* compiler() const = 0; @@ -1179,18 +1181,18 @@ class Device : public RuntimeObject { static std::vector getDevices(cl_device_type type, //!< Device type bool offlineDevices //!< Enable offline devices - ); + ); static size_t numDevices(cl_device_type type, //!< Device type bool offlineDevices //!< Enable offline devices - ); + ); static bool getDeviceIDs(cl_device_type deviceType, //!< Device type cl_uint numEntries, //!< Number of entries in the array cl_device_id* devices, //!< Array of the device ID(s) cl_uint* numDevices, //!< Number of available devices bool offlineDevices //!< Report offline devices - ); + ); const device::Info& info() const { return info_; } @@ -1288,7 +1290,8 @@ class Device : public RuntimeObject { return true; }; - virtual bool SetClockMode(const cl_set_device_clock_mode_input_amd setClockModeInput, cl_set_device_clock_mode_output_amd* pSetClockModeOutput) { + virtual bool SetClockMode(const cl_set_device_clock_mode_input_amd setClockModeInput, + cl_set_device_clock_mode_output_amd* pSetClockModeOutput) { return true; }; //! Returns TRUE if the device is available for computations @@ -1331,21 +1334,24 @@ class Device : public RuntimeObject { // P2P devices that are accessible from the current device std::vector p2pDevices_; -#if defined(WITH_LIGHTNING_COMPILER) && ! defined(USE_COMGR_LIBRARY) + // P2P devices for memory allocation. This list contains devices that can have access to the + // current device + std::vector p2p_access_devices_; + +#if defined(WITH_LIGHTNING_COMPILER) && !defined(USE_COMGR_LIBRARY) amd::CacheCompilation* cacheCompilation() const { return cacheCompilation_.get(); } #endif //! Checks if OCL runtime can use code object manager for compilation bool ValidateComgr(); - virtual amd::Memory *IpcAttach(const void* handle, size_t mem_size, unsigned int flags, void** dev_ptr) const { + virtual amd::Memory* IpcAttach(const void* handle, size_t mem_size, unsigned int flags, + void** dev_ptr) const { ShouldNotReachHere(); return nullptr; } - virtual void IpcDetach(amd::Memory& memory) const { - ShouldNotReachHere(); - } + virtual void IpcDetach(amd::Memory& memory) const { ShouldNotReachHere(); } //! Return private global device context for P2P allocations amd::Context& GlbCtx() const { return *glb_ctx_; } @@ -1356,6 +1362,12 @@ class Device : public RuntimeObject { //! Staging buffer for P2P transfer Memory* P2PStage() const { return p2p_stage_; } + //! Does this device allow P2P access? + bool P2PAccessAllowed() const { return (p2p_access_devices_.size() > 0) ? true : false; } + + //! Returns the list of devices that can have access to the current + const std::vector& P2PAccessDevices() const { return p2p_access_devices_; } + protected: //! Enable the specified extension char* getExtensionString(); @@ -1366,14 +1378,14 @@ class Device : public RuntimeObject { BlitProgram* blitProgram_; //!< Blit program info static AppProfile appProfile_; //!< application profile HwDebugManager* hwDebugMgr_; //!< Hardware Debug manager -#if defined(WITH_LIGHTNING_COMPILER) && ! defined(USE_COMGR_LIBRARY) - //! Compilation with cache support +#if defined(WITH_LIGHTNING_COMPILER) && !defined(USE_COMGR_LIBRARY) + //! Compilation with cache support std::unique_ptr cacheCompilation_; #endif - static amd::Context* glb_ctx_; //!< Global context with all devices - static amd::Monitor p2p_stage_ops_; //!< Lock to serialise cache for the P2P resources - static Memory* p2p_stage_; //!< Staging resources + static amd::Context* glb_ctx_; //!< Global context with all devices + static amd::Monitor p2p_stage_ops_; //!< Lock to serialise cache for the P2P resources + static Memory* p2p_stage_; //!< Staging resources private: bool IsTypeMatching(cl_device_type type, bool offlineDevices); @@ -1388,7 +1400,7 @@ class Device : public RuntimeObject { std::map* vaCacheMap_; //!< VA cache map }; -#if defined(WITH_LIGHTNING_COMPILER) && ! defined(USE_COMGR_LIBRARY) +#if defined(WITH_LIGHTNING_COMPILER) && !defined(USE_COMGR_LIBRARY) //! Compilation process with cache support. class CacheCompilation : public amd::HeapObject { public: @@ -1425,7 +1437,7 @@ class CacheCompilation : public amd::HeapObject { StringCache codeCache_; //! Cached codes const bool isCodeCacheEnabled_; //! Code cache enable }; -#endif // defined(WITH_LIGHTNING_COMPILER) || defined(USE_COMGR_LIBRARY) +#endif // defined(WITH_LIGHTNING_COMPILER) || defined(USE_COMGR_LIBRARY) /*! @} * @} diff --git a/rocclr/runtime/device/pal/paldevice.cpp b/rocclr/runtime/device/pal/paldevice.cpp index 4132b19f78..0faec34786 100644 --- a/rocclr/runtime/device/pal/paldevice.cpp +++ b/rocclr/runtime/device/pal/paldevice.cpp @@ -1082,7 +1082,7 @@ bool Device::create(Pal::IDevice* device) { return true; } -// ===================================================================================================================== +// ================================================================================================ // Master function that handles developer callbacks from PAL. void PAL_STDCALL Device::PalDeveloperCallback(void* pPrivateData, const Pal::uint32 deviceIndex, Pal::Developer::CallbackType type, void* pCbData) { @@ -1333,6 +1333,29 @@ bool Device::init() { } if (!foundDevice) { Device::tearDown(); + } else { + // Loop through all available devices + uint32_t all_devices = amd::Device::numDevices(CL_DEVICE_TYPE_GPU, false); + for (uint32_t device0 = gStartDevice; device0 < all_devices; ++device0) { + // Find all device that can have access to the current device + for (uint32_t device1 = gStartDevice; device1 < all_devices; ++device1) { + // If it's not the same device, then validate P2P settings + if ((devices()[device0] != devices()[device1]) && + static_cast(devices()[device1])->settings().enableHwP2P_) { + Pal::GpuCompatibilityInfo comp_info = {}; + // Can device 0 have access to device1? + static_cast(devices()[device0]) + ->iDev() + ->GetMultiGpuCompatibility(*static_cast(devices()[device1])->iDev(), + &comp_info); + // Check P2P capability + if (comp_info.flags.peerTransfer) { + devices()[device0]->p2pDevices_.push_back(as_cl(devices()[device1])); + devices()[device1]->p2p_access_devices_.push_back(devices()[device0]); + } + } + } + } } return true; } @@ -1516,6 +1539,12 @@ pal::Memory* Device::createBuffer(amd::Memory& owner, bool directAccess) const { params.owner_ = &owner; params.gpu_ = static_cast(owner.getVirtualDevice()); params.svmBase_ = static_cast(owner.svmBase()); + if (owner.P2PAccess()) { + params.svmBase_ = static_cast(owner.BaseP2PMemory()); + if (params.svmBase_ != nullptr) { + type = Resource::P2PAccess; + } + } // Create memory object result = gpuMemory->create(type, ¶ms); @@ -1531,7 +1560,8 @@ pal::Memory* Device::createBuffer(amd::Memory& owner, bool directAccess) const { pipeInit); } // If memory has direct access from host, then get CPU address - if (gpuMemory->isHostMemDirectAccess() && (type != Resource::ExternalPhysical)) { + if (gpuMemory->isHostMemDirectAccess() && (type != Resource::ExternalPhysical) && + (type != Resource::P2PAccess)) { void* address = gpuMemory->map(nullptr); if (address != nullptr) { // Copy saved memory diff --git a/rocclr/runtime/device/pal/palmemory.cpp b/rocclr/runtime/device/pal/palmemory.cpp index bad5652845..c75c2e17bc 100644 --- a/rocclr/runtime/device/pal/palmemory.cpp +++ b/rocclr/runtime/device/pal/palmemory.cpp @@ -119,6 +119,7 @@ bool Memory::create(Resource::MemoryType memType, Resource::CreateParams* params switch (memoryType()) { case Resource::Pinned: case Resource::ExternalPhysical: + case Resource::P2PAccess: // Marks memory object for direct GPU access to the host memory flags_ |= HostMemoryDirectAccess; break; @@ -903,7 +904,7 @@ void Memory::mgpuCacheWriteBack() { amd::ScopedLock lock(owner()->lockMemoryOps()); // Attempt to allocate a staging buffer if don't have any - if (owner()->getHostMem() == nullptr) { + if (!owner()->P2PAccess() && (owner()->getHostMem() == nullptr)) { if (nullptr != owner()->getSvmPtr()) { owner()->commitSvmMemory(); owner()->setHostMem(owner()->getSvmPtr()); diff --git a/rocclr/runtime/device/pal/palresource.cpp b/rocclr/runtime/device/pal/palresource.cpp index 088978846b..b81fd90b18 100644 --- a/rocclr/runtime/device/pal/palresource.cpp +++ b/rocclr/runtime/device/pal/palresource.cpp @@ -152,6 +152,26 @@ GpuMemoryReference* GpuMemoryReference::Create(const Device& dev, return memRef; } +// ================================================================================================ +GpuMemoryReference* GpuMemoryReference::Create(const Device& dev, + const Pal::PeerGpuMemoryOpenInfo& openInfo) { + Pal::Result result; + size_t gpuMemSize = dev.iDev()->GetPeerGpuMemorySize(openInfo, &result); + if (result != Pal::Result::Success) { + return nullptr; + } + + GpuMemoryReference* memRef = new (gpuMemSize) GpuMemoryReference(dev); + if (memRef != nullptr) { + result = dev.iDev()->OpenPeerGpuMemory(openInfo, &memRef[1], &memRef->gpuMem_); + if (result != Pal::Result::Success) { + memRef->release(); + return nullptr; + } + } + return memRef; +} + // ================================================================================================ GpuMemoryReference::GpuMemoryReference(const Device& dev) : gpuMem_(nullptr), cpuAddress_(nullptr), device_(dev), gpu_(nullptr) {} @@ -367,6 +387,7 @@ void Resource::memTypeToHeap(Pal::GpuMemoryCreateInfo* createInfo) { createInfo->heapCount = 2; createInfo->heaps[0] = Pal::GpuHeapLocal; createInfo->heaps[1] = Pal::GpuHeapGartUswc; + createInfo->flags.peerWritable = dev().P2PAccessAllowed(); #ifdef ATI_OS_LINUX // Note: SSG in Linux requires DGMA heap if (dev().properties().gpuMemoryProperties.busAddressableMemSize > 0) { @@ -391,6 +412,7 @@ void Resource::memTypeToHeap(Pal::GpuMemoryCreateInfo* createInfo) { createInfo->heaps[0] = Pal::GpuHeapInvisible; createInfo->heaps[1] = Pal::GpuHeapLocal; createInfo->heaps[2] = Pal::GpuHeapGartUswc; + createInfo->flags.peerWritable = dev().P2PAccessAllowed(); break; default: createInfo->heaps[0] = Pal::GpuHeapLocal; @@ -903,6 +925,20 @@ bool Resource::CreateInterop(CreateParams* params) { return true; } +// ================================================================================================ +bool Resource::CreateP2PAccess(CreateParams* params) { + Pal::PeerGpuMemoryOpenInfo openInfo = {}; + openInfo.pOriginalMem = params->svmBase_->iMem(); + + memRef_ = GpuMemoryReference::Create(dev(), openInfo); + if (nullptr == memRef_) { + return false; + } + desc_.cardMemory_ = false; + offset_ = params->svmBase_->offset(); + return true; +} + // ================================================================================================ bool Resource::CreatePinned(CreateParams* params) { PinnedParams* pinned = reinterpret_cast(params); @@ -1070,42 +1106,45 @@ bool Resource::create(MemoryType memType, CreateParams* params) { if (dev().settings().disablePersistent_ && (memoryType() == Persistent)) { desc_.type_ = RemoteUSWC; } + switch (memoryType()) { + case OGLInterop: + case D3D9Interop: + case D3D10Interop: + case D3D11Interop: + return CreateInterop(params); + case P2PAccess: + return CreateP2PAccess(params); + case Pinned: + return CreatePinned(params); + case View: { + // Save the offset in the global heap + ViewParams* view = reinterpret_cast(params); + offset_ = view->offset_; - if ((memoryType() == OGLInterop) || (memoryType() == D3D9Interop) || - (memoryType() == D3D10Interop) || (memoryType() == D3D11Interop)) { - return CreateInterop(params); + // Make sure parent was provided + if (nullptr != view->resource_) { + viewOwner_ = view->resource_; + offset_ += viewOwner_->offset(); + if (viewOwner_->data() != nullptr) { + address_ = viewOwner_->data() + view->offset_; + mapCount_++; + } + memRef_ = viewOwner_->memRef_; + memRef_->retain(); + desc_.cardMemory_ = viewOwner_->desc().cardMemory_; + } else { + desc_.type_ = Empty; + } + return true; + } + default: + break; } if (!desc_.buffer_) { return CreateImage(params); } - if (memoryType() == Pinned) { - return CreatePinned(params); - } - - if (memoryType() == View) { - // Save the offset in the global heap - ViewParams* view = reinterpret_cast(params); - offset_ = view->offset_; - - // Make sure parent was provided - if (nullptr != view->resource_) { - viewOwner_ = view->resource_; - offset_ += viewOwner_->offset(); - if (viewOwner_->data() != nullptr) { - address_ = viewOwner_->data() + view->offset_; - mapCount_++; - } - memRef_ = viewOwner_->memRef_; - memRef_->retain(); - desc_.cardMemory_ = viewOwner_->desc().cardMemory_; - } else { - desc_.type_ = Empty; - } - return true; - } - Pal::gpusize svmPtr = 0; if ((nullptr != params) && (nullptr != params->owner_) && (nullptr != params->owner_->getSvmPtr())) { @@ -1823,6 +1862,7 @@ bool MemorySubAllocator::CreateChunk(const Pal::IGpuMemory* reserved_va) { createInfo.priority = Pal::GpuMemPriority::Normal; createInfo.heapCount = 1; createInfo.heaps[0] = Pal::GpuHeapInvisible; + createInfo.flags.peerWritable = device_->P2PAccessAllowed(); GpuMemoryReference* mem_ref = GpuMemoryReference::Create(*device_, createInfo); if (mem_ref != nullptr) { return InitAllocator(mem_ref); diff --git a/rocclr/runtime/device/pal/palresource.hpp b/rocclr/runtime/device/pal/palresource.hpp index c2fb0bcad0..857000fdf2 100644 --- a/rocclr/runtime/device/pal/palresource.hpp +++ b/rocclr/runtime/device/pal/palresource.hpp @@ -35,6 +35,8 @@ class GpuMemoryReference : public amd::ReferenceCountedObject { static GpuMemoryReference* Create(const Device& dev, const Pal::ExternalImageOpenInfo& openInfo, Pal::ImageCreateInfo* imgCreateInfo, Pal::IImage** image); + static GpuMemoryReference* Create(const Device& dev, const Pal::PeerGpuMemoryOpenInfo& openInfo); + //! Default constructor GpuMemoryReference(const Device& dev); @@ -53,10 +55,10 @@ class GpuMemoryReference : public amd::ReferenceCountedObject { private: //! Disable copy constructor - GpuMemoryReference(const GpuMemoryReference&); + GpuMemoryReference(const GpuMemoryReference&) = delete; //! Disable operator= - GpuMemoryReference& operator=(const GpuMemoryReference&); + GpuMemoryReference& operator=(const GpuMemoryReference&) = delete; }; static constexpr Pal::gpusize MaxGpuAlignment = 4 * Ki; @@ -144,6 +146,7 @@ class Resource : public amd::HeapObject { D3D9Interop, //!< resource is a D3D9 memory object Scratch, //!< resource is scratch memory Shader, //!< resource is a shader + P2PAccess //!< resource is a shared resource for P2P access }; //! Resource map flags @@ -416,6 +419,13 @@ class Resource : public amd::HeapObject { bool CreateSvm(CreateParams* params, //!< special parameters for resource allocation Pal::gpusize svmPtr); + /*! \brief Creates a PAL P2P object, associated with the resource + * + * \return True if we succesfully created a PAL P2P resource + */ + bool CreateP2PAccess(CreateParams* params //!< special parameters for resource allocation + ); + uint elementSize_; //!< Size of a single element in bytes private: @@ -567,9 +577,9 @@ class ResourceCache : public amd::HeapObject { amd::Monitor lockCacheOps_; //!< Lock to serialise cache access - size_t cacheSize_; //!< Current cache size in bytes - size_t lclCacheSize_; //!< Local memory stored in the cache - const size_t cacheSizeLimit_; //!< Cache size limit in bytes + size_t cacheSize_; //!< Current cache size in bytes + size_t lclCacheSize_; //!< Local memory stored in the cache + const size_t cacheSizeLimit_; //!< Cache size limit in bytes //! PAL resource cache std::list > resCache_; diff --git a/rocclr/runtime/device/pal/palsettings.cpp b/rocclr/runtime/device/pal/palsettings.cpp index 131bb4afed..0cbe3a17a8 100644 --- a/rocclr/runtime/device/pal/palsettings.cpp +++ b/rocclr/runtime/device/pal/palsettings.cpp @@ -29,7 +29,6 @@ struct ModifyMaxWorkload { #endif }; - Settings::Settings() { // Initialize the GPU device default settings oclVersion_ = OpenCL12; @@ -144,6 +143,7 @@ Settings::Settings() { enableWave32Mode_ = false; hsailExplicitXnack_ = false; lcWavefrontSize64_ = true; + enableHwP2P_ = false; } bool Settings::create(const Pal::DeviceProperties& palProp, @@ -213,6 +213,8 @@ bool Settings::create(const Pal::DeviceProperties& palProp, lcWavefrontSize64_ = !enableWave32Mode_; // Fall through to AI (gfx9) ... case Pal::AsicRevision::Vega20: + // Enable HW P2P path for Vega20+. Runtime still relies on KMD/PAL for support + enableHwP2P_ = true; case Pal::AsicRevision::Vega12: case Pal::AsicRevision::Vega10: case Pal::AsicRevision::Raven: @@ -515,6 +517,10 @@ void Settings::override() { resourceCacheSize_ = GPU_RESOURCE_CACHE_SIZE * Mi; } + if (!flagIsDefault(GPU_ENABLE_HW_P2P)) { + enableHwP2P_ = GPU_ENABLE_HW_P2P; + } + if (!flagIsDefault(AMD_GPU_FORCE_SINGLE_FP_DENORM)) { switch (AMD_GPU_FORCE_SINGLE_FP_DENORM) { case 0: diff --git a/rocclr/runtime/device/pal/palsettings.hpp b/rocclr/runtime/device/pal/palsettings.hpp index 6b8ee86768..716b45f307 100644 --- a/rocclr/runtime/device/pal/palsettings.hpp +++ b/rocclr/runtime/device/pal/palsettings.hpp @@ -61,7 +61,8 @@ class Settings : public device::Settings { uint rgpSqttWaitIdle_ : 1; //!< Wait for idle after SQTT trace uint rgpSqttForceDisable_ : 1; //!< Disables SQTT uint splitSizeForWin7_ : 1; //!< DMA flush split size for Win 7 - uint reserved_ : 11; + uint enableHwP2P_ : 1; //!< Forces HW P2P path for testing + uint reserved_ : 10; }; uint value_; }; diff --git a/rocclr/runtime/device/pal/palvirtual.cpp b/rocclr/runtime/device/pal/palvirtual.cpp index ff8bffefae..8ea89b9f48 100644 --- a/rocclr/runtime/device/pal/palvirtual.cpp +++ b/rocclr/runtime/device/pal/palvirtual.cpp @@ -821,13 +821,12 @@ bool VirtualGPU::create(bool profiling, uint deviceQueueSize, uint rtCUs, // Check if device has SDMA engines if (dev().numDMAEngines() != 0 && !PAL_DISABLE_SDMA) { uint sdma; - // If only 1 DMA engine is available then use that one - if ((dev().numDMAEngines() < 2) || ((idx & 0x1) && !dev().settings().svmFineGrainSystem_)) { + // If only 1 SDMA engine is available then use that one, otherwise it's a round-robin manner + if ((dev().numDMAEngines() < 2) || ((idx + 1) & 0x1)) { sdma = 0; } else { sdma = 1; } - queues_[SdmaEngine] = Queue::Create( *this, Pal::QueueTypeDma, sdma, cmdAllocator_, amd::CommandQueue::RealTimeDisabled, amd::CommandQueue::Priority::Normal, residency_limit, max_cmd_buffers); @@ -843,7 +842,8 @@ bool VirtualGPU::create(bool profiling, uint deviceQueueSize, uint rtCUs, } } } else { - Unimplemented(); + LogError("Runtme couldn't find compute queues!"); + return false; } if (!managedBuffer_.create(Resource::RemoteUSWC)) { @@ -1695,22 +1695,22 @@ void VirtualGPU::submitCopyMemoryP2P(amd::CopyMemoryP2PCommand& cmd) { profilingBegin(cmd); - Memory* srcDevMem = static_cast( - cmd.source().getDeviceMemory(*cmd.source().getContext().devices()[0])); - Memory* dstDevMem = static_cast( - cmd.destination().getDeviceMemory(*cmd.destination().getContext().devices()[0])); + // Get the device memory objects for the current device + Memory* srcDevMem = dev().getGpuMemory(&cmd.source()); + Memory* dstDevMem = dev().getGpuMemory(&cmd.destination()); - bool p2pAllowed = false; -#if 0 - // Loop through all available P2P devices for the destination buffer - for (auto agent: dstDevMem->dev().p2pAgents()) { - // Find the device, which is matching the current - if (agent.handle == dev().getBackendDevice().handle) { - p2pAllowed = true; - break; - } + bool p2pAllowed = true; + + // If any device object is null, then no HW P2P and runtime has to use staging + if (srcDevMem == nullptr) { + srcDevMem = static_cast( + cmd.source().getDeviceMemory(*cmd.source().getContext().devices()[0])); + p2pAllowed = false; + } else if (dstDevMem == nullptr) { + dstDevMem = static_cast( + cmd.destination().getDeviceMemory(*cmd.destination().getContext().devices()[0])); + p2pAllowed = false; } -#endif // Synchronize source and destination memory device::Memory::SyncFlags syncFlags; diff --git a/rocclr/runtime/device/rocm/rocdevice.cpp b/rocclr/runtime/device/rocm/rocdevice.cpp index 1f4d0081e7..4052daa821 100644 --- a/rocclr/runtime/device/rocm/rocdevice.cpp +++ b/rocclr/runtime/device/rocm/rocdevice.cpp @@ -1013,7 +1013,7 @@ bool Device::populateOCLDeviceConstants() { for (auto agent: gpu_agents_) { if (agent.handle != _bkendDevice.handle) { hsa_status_t err; - // Can current GPU have access to another GPU memory pool + // Can another GPU (agent) have access to the current GPU memory pool (gpuvm_segment_)? hsa_amd_memory_pool_access_t access; err = hsa_amd_agent_memory_pool_get_info(agent, gpuvm_segment_, HSA_AMD_AGENT_MEMORY_POOL_INFO_ACCESS, &access); if (err != HSA_STATUS_SUCCESS) { @@ -1023,6 +1023,7 @@ bool Device::populateOCLDeviceConstants() { // Find accessible p2p agents - i.e != HSA_AMD_MEMORY_POOL_ACCESS_NEVER_ALLOWED if (HSA_AMD_MEMORY_POOL_ACCESS_ALLOWED_BY_DEFAULT == access || HSA_AMD_MEMORY_POOL_ACCESS_DISALLOWED_BY_DEFAULT == access) { + // Agent can have access to the current gpuvm_segment_ p2p_agents_.push_back(agent); } } diff --git a/rocclr/runtime/platform/command.cpp b/rocclr/runtime/platform/command.cpp index c6ae383f5d..034a3182e7 100644 --- a/rocclr/runtime/platform/command.cpp +++ b/rocclr/runtime/platform/command.cpp @@ -552,7 +552,14 @@ bool CopyMemoryP2PCommand::validateMemory() { LogPrintfError("Can't allocate memory size - 0x%08X bytes!", memory2_->getSize()); return false; } - if (devices[0]->P2PStage() != nullptr) { + bool p2pStaging = false; + // Validate P2P memories on the current device, if any of them is null, then it's p2p staging + if ((nullptr == memory1_->getDeviceMemory(queue()->device())) || + (nullptr == memory2_->getDeviceMemory(queue()->device()))) { + p2pStaging = true; + } + + if (devices[0]->P2PStage() != nullptr && p2pStaging) { amd::ScopedLock lock(devices[0]->P2PStageOps()); // Make sure runtime allocates memory on every device for (uint d = 0; d < devices[0]->GlbCtx().devices().size(); ++d) { diff --git a/rocclr/runtime/platform/memory.cpp b/rocclr/runtime/platform/memory.cpp index b1efa4bbba..c8fa903a15 100644 --- a/rocclr/runtime/platform/memory.cpp +++ b/rocclr/runtime/platform/memory.cpp @@ -70,14 +70,14 @@ Memory::Memory(Context& context, Type type, Flags flags, size_t size, void* svmP version_(0), lastWriter_(NULL), interopObj_(NULL), - isParent_(false), vDev_(NULL), - forceSysMemAlloc_(false), mapCount_(0), svmHostAddress_(svmPtr), - svmPtrCommited_(flags & CL_MEM_SVM_FINE_GRAIN_BUFFER ? true : false), - canBeCached_(true), - lockMemoryOps_("Memory Ops Lock", true) {} + flagsEx_(0), + lockMemoryOps_("Memory Ops Lock", true) { + svmPtrCommited_ = (flags & CL_MEM_SVM_FINE_GRAIN_BUFFER) ? true : false; + canBeCached_ = true; +} Memory::Memory(Memory& parent, Flags flags, size_t origin, size_t size, Type type) : numDevices_(0), @@ -93,14 +93,13 @@ Memory::Memory(Memory& parent, Flags flags, size_t origin, size_t size, Type typ version_(parent.getVersion()), lastWriter_(parent.getLastWriter()), interopObj_(parent.getInteropObj()), - isParent_(false), vDev_(NULL), - forceSysMemAlloc_(false), mapCount_(0), svmHostAddress_(parent.getSvmPtr()), - svmPtrCommited_(parent.isSvmPtrCommited()), - canBeCached_(true), + flagsEx_(0), lockMemoryOps_("Memory Ops Lock", true) { + svmPtrCommited_ = parent.isSvmPtrCommited(); + canBeCached_ = true; parent_->retain(); parent_->isParent_ = true; @@ -126,13 +125,31 @@ Memory::Memory(Memory& parent, Flags flags, size_t origin, size_t size, Type typ } } +uint32_t Memory::NumDevicesWithP2P() { + uint32_t devices = context_().devices().size(); + if (devices == 1) { + // Add p2p devices for allocation + devices = context_().devices().size() + context_().devices()[0]->P2PAccessDevices().size(); + if (devices > 1) { + p2pAccess_ = true; + } + } + return devices; +} + void Memory::initDeviceMemory() { deviceMemories_ = reinterpret_cast(reinterpret_cast(this) + sizeof(Memory)); - memset(deviceMemories_, 0, context_().devices().size() * sizeof(DeviceMemory)); + memset(deviceMemories_, 0, NumDevicesWithP2P() * sizeof(DeviceMemory)); } void* Memory::operator new(size_t size, const Context& context) { - return RuntimeObject::operator new(size + context.devices().size() * sizeof(DeviceMemory)); + uint32_t devices = context.devices().size(); + if (devices == 1) { + // Add p2p devices for allocation + devices = context.devices().size() + context.devices()[0]->P2PAccessDevices().size(); + } + + return RuntimeObject::operator new(size + devices * sizeof(DeviceMemory)); } void Memory::operator delete(void* p) { RuntimeObject::operator delete(p); } @@ -259,7 +276,14 @@ bool Memory::addDeviceMemory(const Device* dev) { bool result = false; AllocState create = AllocCreate; AllocState init = AllocInit; + if (make_atomic(deviceAlloced_[dev]).compareAndSet(init, create)) { + // Check if runtime already allocated all available slots for device memory + if (numDevices() == NumDevicesWithP2P()) { + // Mark the allocation as an empty + deviceAlloced_[dev] = AllocInit; + return false; + } device::Memory* dm = dev->createMemory(*this); // Add the new memory allocation to the device map @@ -267,7 +291,7 @@ bool Memory::addDeviceMemory(const Device* dev) { deviceMemories_[numDevices_].ref_ = dev; deviceMemories_[numDevices_].value_ = dm; numDevices_++; - assert((numDevices() <= context_().devices().size()) && "Too many device objects"); + assert((numDevices() <= NumDevicesWithP2P()) && "Too many device objects"); // Mark the allocation with the complete flag deviceAlloced_[dev] = AllocComplete; @@ -275,6 +299,7 @@ bool Memory::addDeviceMemory(const Device* dev) { svmBase_ = dm; } } else { + LogError("Video memory allocation failed!"); // Mark the allocation as an empty deviceAlloced_[dev] = AllocInit; } @@ -322,7 +347,6 @@ device::Memory* Memory::getDeviceMemory(const Device& dev, bool alloc) { if ((NULL == dm) && alloc) { if (!addDeviceMemory(&dev)) { - LogError("Video memory allocation failed!"); return NULL; } dm = deviceMemories_[numDevices() - 1].value_; @@ -446,7 +470,7 @@ void Memory::uncommitSvmMemory() { void Buffer::initDeviceMemory() { deviceMemories_ = reinterpret_cast(reinterpret_cast(this) + sizeof(Buffer)); - memset(deviceMemories_, 0, context_().devices().size() * sizeof(DeviceMemory)); + memset(deviceMemories_, 0, NumDevicesWithP2P() * sizeof(DeviceMemory)); } bool Buffer::create(void* initFrom, bool sysMemAlloc, bool skipAlloc) { @@ -472,7 +496,7 @@ bool Buffer::validateRegion(const Coord3D& origin, const Coord3D& region) const void Pipe::initDeviceMemory() { deviceMemories_ = reinterpret_cast(reinterpret_cast(this) + sizeof(Pipe)); - memset(deviceMemories_, 0, context_().devices().size() * sizeof(DeviceMemory)); + memset(deviceMemories_, 0, NumDevicesWithP2P() * sizeof(DeviceMemory)); } #define GETMIPDIM(dim, mip) (((dim >> mip) > 0) ? (dim >> mip) : 1) @@ -631,7 +655,7 @@ void Image::initDimension() { void Image::initDeviceMemory() { deviceMemories_ = reinterpret_cast(reinterpret_cast(this) + sizeof(Image)); - memset(deviceMemories_, 0, context_().devices().size() * sizeof(DeviceMemory)); + memset(deviceMemories_, 0, NumDevicesWithP2P() * sizeof(DeviceMemory)); } bool Image::create(void* initFrom) { return Memory::create(initFrom); } diff --git a/rocclr/runtime/platform/memory.hpp b/rocclr/runtime/platform/memory.hpp index 75b0254fa9..7711c19afa 100644 --- a/rocclr/runtime/platform/memory.hpp +++ b/rocclr/runtime/platform/memory.hpp @@ -24,7 +24,7 @@ namespace device { class Memory; class VirtualDevice; -} +} // namespace device namespace amd { @@ -42,7 +42,7 @@ struct BufferRect : public amd::EmbeddedObject { const size_t* region, //!< Copy region size_t bufferRowPitch, //!< Provided buffer's row pitch size_t bufferSlicePitch //!< Provided buffer's slice pitch - ); + ); //! Returns the plain offset for the (X, Y, Z) location size_t offset(size_t x, //!< Coordinate in X dimension @@ -120,6 +120,9 @@ class Memory : public amd::RuntimeObject { typedef cl_mem_flags Flags; typedef DeviceMap DeviceMemory; + //! Returns the number of devices this memory object is associated, including P2P access + uint32_t NumDevicesWithP2P(); + size_t numDevices_; //!< Number of devices //! The device memory objects included in this memory @@ -141,13 +144,19 @@ class Memory : public amd::RuntimeObject { size_t version_; //!< Update count, used for coherency const Device* lastWriter_; //!< Which device wrote most recently (NULL if host) InteropObject* interopObj_; //!< Interop object - bool isParent_; //!< This object is a parent device::VirtualDevice* vDev_; //!< Memory object belongs to a virtual device only - bool forceSysMemAlloc_; //!< Forces system memory allocation std::atomic_uint mapCount_; //!< Keep track of number of mappings for a memory object void* svmHostAddress_; //!< svm host address; - bool svmPtrCommited_; //!< svm host address committed flag; - bool canBeCached_; //!< flag to if the object can be cached; + union { + struct { + uint32_t isParent_ : 1; //!< This object is a parent + uint32_t forceSysMemAlloc_ : 1; //!< Forces system memory allocation + uint32_t svmPtrCommited_ : 1; //!< svm host address committed flag + uint32_t canBeCached_ : 1; //!< flag to if the object can be cached + uint32_t p2pAccess_ : 1; //!< Memory object allows P2P access + }; + uint32_t flagsEx_; + }; private: //! Disable default assignment operator @@ -168,20 +177,20 @@ class Memory : public amd::RuntimeObject { Flags flags, //!< Object's flags size_t size, //!< Memory size void* svmPtr = NULL //!< svm host memory address, NULL if no SVM mem object - ); + ); Memory(Memory& parent, //!< Context object Flags flags, //!< Object's flags size_t offset, //!< Memory offset size_t size, //!< Memory size Type type = 0 //!< Memory type - ); + ); //! Memory object destructor virtual ~Memory(); //! Copies initialization data to the backing store virtual void copyToBackingStore(void* initFrom //!< Pointer to the initialization memory - ); + ); //! Initializes the device memory array virtual void initDeviceMemory(); @@ -193,14 +202,14 @@ class Memory : public amd::RuntimeObject { //! Placement new operator. void* operator new(size_t size, //!< Original allocation size const Context& context //!< Context this memory object is allocated in. - ); + ); // Provide a "matching" placement delete operator. void operator delete(void*, //!< Pointer to deallocate const Context& context //!< Context this memory object is allocated in. - ); + ); // and a regular delete operator to satisfy synthesized methods. void operator delete(void* //!< Pointer to deallocate - ); + ); //! Returns the memory lock object amd::Monitor& lockMemoryOps() { return lockMemoryOps_; } @@ -228,32 +237,32 @@ class Memory : public amd::RuntimeObject { virtual Pipe* asPipe() { return NULL; } //! Creates and initializes device (cache) memory for all devices - virtual bool create(void* initFrom = NULL, //!< Pointer to the initialization data - bool sysMemAlloc = false, //!< Allocate device memory in system memory - bool skipAlloc = false //!< Skip device memory allocation - ); + virtual bool create(void* initFrom = NULL, //!< Pointer to the initialization data + bool sysMemAlloc = false, //!< Allocate device memory in system memory + bool skipAlloc = false //!< Skip device memory allocation + ); //! Allocates device (cache) memory for a specific device bool addDeviceMemory(const Device* dev //!< Device object - ); + ); //! Replaces device (cache) memory for a specific device void replaceDeviceMemory(const Device* dev, //!< Device object device::Memory* dm //!< New device memory object for replacement - ); + ); //! Find the section for the given device. Return NULL if not found. device::Memory* getDeviceMemory(const Device& dev, //!< Device object bool alloc = true //!< Allocates memory - ); + ); //! Allocate host memory (as required) bool allocHostMemory(void* initFrom, //!< Host memory provided by the application bool allocHostMem, //!< Force system memory allocation bool forceCopy = false //!< Force system memory allocation - ); + ); - virtual void IpcCreate (size_t offset, size_t* mem_size, void* handle) const { + virtual void IpcCreate(size_t offset, size_t* mem_size, void* handle) const { ShouldNotReachHere(); } @@ -314,8 +323,14 @@ class Memory : public amd::RuntimeObject { void uncommitSvmMemory(); void setCacheStatus(bool canBeCached) { canBeCached_ = canBeCached; - } //!< set the memobject cached status; - bool canBeCached() const { return canBeCached_; } //!< get the memobject cached status; + } //!< set the memobject cached status + bool canBeCached() const { return canBeCached_; } //!< get the memobject cached status + + //! Check if this objects allows P2P access + bool P2PAccess() const { return p2pAccess_; } + + //! Returns the base device memory object for possible P2P access + device::Memory* BaseP2PMemory() const { return deviceMemories_[0].value_; } device::Memory* svmBase() const { return svmBase_; } //!< Returns SVM base for MGPU case }; @@ -339,10 +354,10 @@ class Buffer : public Memory { Buffer(Memory& parent, Flags flags, size_t origin, size_t size) : Memory(parent, flags, origin, size) {} - bool create(void* initFrom = NULL, //!< Pointer to the initialization data - bool sysMemAlloc = false, //!< Allocate device memory in system memory - bool skipAlloc = false //!< Skip device memory allocation - ); + bool create(void* initFrom = NULL, //!< Pointer to the initialization data + bool sysMemAlloc = false, //!< Allocate device memory in system memory + bool skipAlloc = false //!< Skip device memory allocation + ); //! static_cast to Buffer with sanity check virtual Buffer* asBuffer() { return this; } @@ -463,7 +478,7 @@ class Image : public Memory { //! Copies initialization data to the backing store virtual void copyToBackingStore(void* initFrom //!< Pointer to the initialization memory - ); + ); void initDimension(); @@ -482,7 +497,7 @@ class Image : public Memory { size_t height, //!< Image height size_t depth, //!< Image depth size_t arraySize //!< Image array size - ); + ); const Format& getImageFormat() const { return impl_.format_; } @@ -512,7 +527,7 @@ class Image : public Memory { device::VirtualDevice* vDev, //!< Virtual device object uint baseMipLevel = 0, //!< Base mip level for a view cl_mem_flags flags = 0 //!< Memory allocation flags - ); + ); //! Returns the impl for this image. Impl& getImpl() { return impl_; } @@ -554,7 +569,7 @@ class Image : public Memory { //! Creates and initializes device (cache) memory for all devices bool create(void* initFrom = NULL //!< Pointer to the initialization data - ); + ); }; //! SVM-related functionality. diff --git a/rocclr/runtime/utils/flags.hpp b/rocclr/runtime/utils/flags.hpp index f346f07985..ec357bd826 100644 --- a/rocclr/runtime/utils/flags.hpp +++ b/rocclr/runtime/utils/flags.hpp @@ -168,17 +168,19 @@ release_on_stg(bool, PAL_DISABLE_SDMA, false, \ release(uint, PAL_RGP_DISP_COUNT, 50, \ "The number of dispatches for RGP capture with SQTT") \ release(bool, GPU_ENABLE_WAVE32_MODE, true, \ - "Enables Wave32 compilation in HW if available") \ + "Enables Wave32 compilation in HW if available") \ release(bool, GPU_ENABLE_LC, IS_LIGHTNING, \ "Enables LC path") \ +release(bool, GPU_ENABLE_HW_P2P, false, \ + "Enables HW P2P path") \ release(uint, GPU_MAX_COMMAND_BUFFERS, 8, \ "The maximum number of command buffers allocated per queue") \ -release(cstring, HIP_VISIBLE_DEVICES, "", \ +release(cstring, HIP_VISIBLE_DEVICES, "", \ + "Only devices whose index is present in the sequence are visible to HIP") \ +release(cstring, CUDA_VISIBLE_DEVICES, "", \ "Only devices whose index is present in the sequence are visible to HIP") \ -release(cstring, CUDA_VISIBLE_DEVICES, "", \ - "Only devices whose index is present in the sequence are visible to HIP") \ -release(bool, GPU_ENABLE_WGP_MODE, true, \ - "Enables WGP Mode in HW if available") \ +release(bool, GPU_ENABLE_WGP_MODE, true, \ + "Enables WGP Mode in HW if available") \ namespace amd {