AI/rocm-systems
2
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

P4 to Git Change 1781985 by gandryey@gera-win10 on 2019/05/13 11:54:40

Просмотр исходного кода 
SWDEV-189140 - Add P2P support in PAL path
	- PAL requires P2P resource open on the usage device. Add the new interface to open the resource
	- Add a hidden P2P device object creation into amd::Memory. It can be activated with OCL context that has a single device.

Affected files ...

... //depot/stg/opencl/drivers/opencl/api/opencl/amdocl/cl_p2p_amd.cpp#2 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/device.hpp#337 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/paldevice.cpp#134 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palmemory.cpp#25 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palresource.cpp#74 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palresource.hpp#28 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palsettings.cpp#80 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palsettings.hpp#23 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palvirtual.cpp#133 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocdevice.cpp#126 edit
... //depot/stg/opencl/drivers/opencl/runtime/platform/command.cpp#93 edit
... //depot/stg/opencl/drivers/opencl/runtime/platform/memory.cpp#136 edit
... //depot/stg/opencl/drivers/opencl/runtime/platform/memory.hpp#109 edit
... //depot/stg/opencl/drivers/opencl/runtime/utils/flags.hpp#306 edit
Этот коммит содержится в:
foreman
2019-05-13 12:19:10 -04:00
родитель 32135bf5cf
Коммит e16de03a2e
13 изменённых файлов: 306 добавлений и 157 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы
rocclr/runtime/device/device.hpp
+57 -45
Просмотреть файл
@@ -19,10 +19,10 @@
#include "devkernel.hpp"
#include "amdocl/cl_profile_amd.h"
#if defined(WITH_LIGHTNING_COMPILER) && ! defined(USE_COMGR_LIBRARY)
#if defined(WITH_LIGHTNING_COMPILER) && !defined(USE_COMGR_LIBRARY)
#include "caching/cache.hpp"
#include "driver/AmdCompiler.h"
#endif // defined(WITH_LIGHTNING_COMPILER) && ! defined(USE_COMGR_LIBRARY)
#endif // defined(WITH_LIGHTNING_COMPILER) && ! defined(USE_COMGR_LIBRARY)
#include "acl.h"
#include "hwdebug.hpp"
@@ -77,9 +77,9 @@ struct Coord3D;
namespace option {
class Options;
} // option
} // namespace option
}
} // namespace amd
enum OclExtensions {
ClKhrFp64 = 0,
@@ -645,13 +645,13 @@ class Memory : public amd::HeapObject {
uint mapFlags, //!< Map flags
size_t* rowPitch = NULL, //!< Row pitch for the mapped memory
size_t* slicePitch = NULL //!< Slice for the mapped memory
) {
) {
return NULL;
}
virtual bool pinSystemMemory(void* hostPtr, //!< System memory address
size_t size //!< Size of allocated system memory
) {
) {
return true;
}
@@ -669,7 +669,7 @@ class Memory : public amd::HeapObject {
uint numLayers = 0, //!< End layer for multilayer map
size_t* rowPitch = NULL, //!< Row pitch for the device memory
size_t* slicePitch = NULL //!< Slice pitch for the device memory
) {
) {
amd::Image* image = owner()->asImage();
if (image != NULL) {
*rowPitch = image->getRowPitch();
@@ -681,7 +681,7 @@ class Memory : public amd::HeapObject {
//! Unmap the device memory
virtual void cpuUnmap(VirtualDevice& vDev //!< Virtual device for unmap operaiton
) {}
) {}
//! Saves map info for this object
//! @note: It's not a thread safe operation, the app must implement
@@ -692,7 +692,7 @@ class Memory : public amd::HeapObject {
uint mapFlags, //!< Map flags
bool entire, //!< True if the enitre memory was mapped
amd::Image* baseMip = nullptr //!< The base mip level for map
);
);
const WriteMapInfo* writeMapInfo(const void* mapAddress) const {
// Unmap must be serialized.
@@ -762,9 +762,10 @@ class Memory : public amd::HeapObject {
//! NB, the map data below is for an API-level map (from clEnqueueMapBuffer),
//! not a physical map. When a memory object does not use USE_HOST_PTR we
//! can use a remote resource and DMA, avoiding the additional CPU memcpy.
amd::Memory* mapMemory_; //!< Memory used as map target buffer
volatile size_t indirectMapCount_; //!< Number of maps
std::unordered_map<const void*, WriteMapInfo> writeMapInfo_; //!< Saved write map info for partial unmap
amd::Memory* mapMemory_; //!< Memory used as map target buffer
volatile size_t indirectMapCount_; //!< Number of maps
std::unordered_map<const void*, WriteMapInfo>
writeMapInfo_; //!< Saved write map info for partial unmap
//! Increment map count
void incIndMapCount() { ++indirectMapCount_; }
@@ -772,7 +773,7 @@ class Memory : public amd::HeapObject {
//! Decrement map count
virtual void decIndMapCount() {}
size_t size_; //!< Memory size
size_t size_; //!< Memory size
private:
//! Disable default copy constructor
@@ -797,8 +798,8 @@ class Sampler : public amd::HeapObject {
const address hwState() const { return hwState_; }
protected:
uint64_t hwSrd_; //!< Device specific HW state for the sampler
address hwState_; //!< CPU pointer to HW state
uint64_t hwSrd_; //!< Device specific HW state for the sampler
address hwState_; //!< CPU pointer to HW state
private:
//! Disable default copy constructor
@@ -873,11 +874,11 @@ class ClBinary : public amd::HeapObject {
//! Store compile options into OCL binary file
void storeCompileOptions(const std::string& compileOptions //!< the compile options to be stored
);
);
//! Store link options into OCL binary file
void storeLinkOptions(const std::string& linkOptions //!< the link options to be stored
);
);
//! Check if the binary is recompilable
bool isRecompilable(std::string& llvmBinary, amd::OclElf::oclElfPlatform thePlatform);
@@ -967,11 +968,11 @@ class ClBinary : public amd::HeapObject {
void release();
const char* binary_; //!< binary data
size_t size_; //!< binary size
uint flags_; //!< CL binary object flags
size_t size_; //!< binary size
uint flags_; //!< CL binary object flags
const char* origBinary_; //!< original binary data
size_t origSize_; //!< original binary size
size_t origSize_; //!< original binary size
int encryptCode_; //!< Encryption Code for input binary (0 for not encrypted)
@@ -1110,21 +1111,22 @@ class MemObjMap : public AllStatic {
public:
static size_t size(); //!< obtain the size of the container
static void AddMemObj(const void* k,
amd::Memory* v); //!< add the host mem pointer and buffer in the container
amd::Memory* v); //!< add the host mem pointer and buffer in the container
static void RemoveMemObj(const void* k); //!< Remove an entry of mem object from the container
static amd::Memory* FindMemObj(
const void* k); //!< find the mem object based on the input pointer
private:
static std::map<uintptr_t, amd::Memory*> MemObjMap_; //!< the mem object<->hostptr information container
static amd::Monitor AllocatedLock_; //!< amd monitor locker
static std::map<uintptr_t, amd::Memory*>
MemObjMap_; //!< the mem object<->hostptr information container
static amd::Monitor AllocatedLock_; //!< amd monitor locker
};
/*! \addtogroup Runtime
* @{
*
* \addtogroup Device Device Abstraction
* @{
*/
* @{
*
* \addtogroup Device Device Abstraction
* @{
*/
class Device : public RuntimeObject {
protected:
typedef aclCompiler Compiler;
@@ -1144,7 +1146,7 @@ class Device : public RuntimeObject {
bool create(Device* device, //!< Device object
const char* extraKernel = NULL, //!< Extra kernels from the device layer
const char* extraOptions = NULL //!< Extra compilation options
);
);
};
virtual Compiler* compiler() const = 0;
@@ -1179,18 +1181,18 @@ class Device : public RuntimeObject {
static std::vector<Device*> getDevices(cl_device_type type, //!< Device type
bool offlineDevices //!< Enable offline devices
);
);
static size_t numDevices(cl_device_type type, //!< Device type
bool offlineDevices //!< Enable offline devices
);
);
static bool getDeviceIDs(cl_device_type deviceType, //!< Device type
cl_uint numEntries, //!< Number of entries in the array
cl_device_id* devices, //!< Array of the device ID(s)
cl_uint* numDevices, //!< Number of available devices
bool offlineDevices //!< Report offline devices
);
);
const device::Info& info() const { return info_; }
@@ -1288,7 +1290,8 @@ class Device : public RuntimeObject {
return true;
};
virtual bool SetClockMode(const cl_set_device_clock_mode_input_amd setClockModeInput, cl_set_device_clock_mode_output_amd* pSetClockModeOutput) {
virtual bool SetClockMode(const cl_set_device_clock_mode_input_amd setClockModeInput,
cl_set_device_clock_mode_output_amd* pSetClockModeOutput) {
return true;
};
//! Returns TRUE if the device is available for computations
@@ -1331,21 +1334,24 @@ class Device : public RuntimeObject {
// P2P devices that are accessible from the current device
std::vector<cl_device_id> p2pDevices_;
#if defined(WITH_LIGHTNING_COMPILER) && ! defined(USE_COMGR_LIBRARY)
// P2P devices for memory allocation. This list contains devices that can have access to the
// current device
std::vector<Device*> p2p_access_devices_;
#if defined(WITH_LIGHTNING_COMPILER) && !defined(USE_COMGR_LIBRARY)
amd::CacheCompilation* cacheCompilation() const { return cacheCompilation_.get(); }
#endif
//! Checks if OCL runtime can use code object manager for compilation
bool ValidateComgr();
virtual amd::Memory *IpcAttach(const void* handle, size_t mem_size, unsigned int flags, void** dev_ptr) const {
virtual amd::Memory* IpcAttach(const void* handle, size_t mem_size, unsigned int flags,
void** dev_ptr) const {
ShouldNotReachHere();
return nullptr;
}
virtual void IpcDetach(amd::Memory& memory) const {
ShouldNotReachHere();
}
virtual void IpcDetach(amd::Memory& memory) const { ShouldNotReachHere(); }
//! Return private global device context for P2P allocations
amd::Context& GlbCtx() const { return *glb_ctx_; }
@@ -1356,6 +1362,12 @@ class Device : public RuntimeObject {
//! Staging buffer for P2P transfer
Memory* P2PStage() const { return p2p_stage_; }
//! Does this device allow P2P access?
bool P2PAccessAllowed() const { return (p2p_access_devices_.size() > 0) ? true : false; }
//! Returns the list of devices that can have access to the current
const std::vector<Device*>& P2PAccessDevices() const { return p2p_access_devices_; }
protected:
//! Enable the specified extension
char* getExtensionString();
@@ -1366,14 +1378,14 @@ class Device : public RuntimeObject {
BlitProgram* blitProgram_; //!< Blit program info
static AppProfile appProfile_; //!< application profile
HwDebugManager* hwDebugMgr_; //!< Hardware Debug manager
#if defined(WITH_LIGHTNING_COMPILER) && ! defined(USE_COMGR_LIBRARY)
//! Compilation with cache support
#if defined(WITH_LIGHTNING_COMPILER) && !defined(USE_COMGR_LIBRARY)
//! Compilation with cache support
std::unique_ptr<amd::CacheCompilation> cacheCompilation_;
#endif
static amd::Context* glb_ctx_; //!< Global context with all devices
static amd::Monitor p2p_stage_ops_; //!< Lock to serialise cache for the P2P resources
static Memory* p2p_stage_; //!< Staging resources
static amd::Context* glb_ctx_; //!< Global context with all devices
static amd::Monitor p2p_stage_ops_; //!< Lock to serialise cache for the P2P resources
static Memory* p2p_stage_; //!< Staging resources
private:
bool IsTypeMatching(cl_device_type type, bool offlineDevices);
@@ -1388,7 +1400,7 @@ class Device : public RuntimeObject {
std::map<uintptr_t, device::Memory*>* vaCacheMap_; //!< VA cache map
};
#if defined(WITH_LIGHTNING_COMPILER) && ! defined(USE_COMGR_LIBRARY)
#if defined(WITH_LIGHTNING_COMPILER) && !defined(USE_COMGR_LIBRARY)
//! Compilation process with cache support.
class CacheCompilation : public amd::HeapObject {
public:
@@ -1425,7 +1437,7 @@ class CacheCompilation : public amd::HeapObject {
StringCache codeCache_; //! Cached codes
const bool isCodeCacheEnabled_; //! Code cache enable
};
#endif // defined(WITH_LIGHTNING_COMPILER) || defined(USE_COMGR_LIBRARY)
#endif // defined(WITH_LIGHTNING_COMPILER) || defined(USE_COMGR_LIBRARY)
/*! @}
* @}
rocclr/runtime/device/pal/paldevice.cpp
+32 -2
Просмотреть файл
@@ -1082,7 +1082,7 @@ bool Device::create(Pal::IDevice* device) {
return true;
}
// =====================================================================================================================
// ================================================================================================
// Master function that handles developer callbacks from PAL.
void PAL_STDCALL Device::PalDeveloperCallback(void* pPrivateData, const Pal::uint32 deviceIndex,
Pal::Developer::CallbackType type, void* pCbData) {
@@ -1333,6 +1333,29 @@ bool Device::init() {
}
if (!foundDevice) {
Device::tearDown();
} else {
// Loop through all available devices
uint32_t all_devices = amd::Device::numDevices(CL_DEVICE_TYPE_GPU, false);
for (uint32_t device0 = gStartDevice; device0 < all_devices; ++device0) {
// Find all device that can have access to the current device
for (uint32_t device1 = gStartDevice; device1 < all_devices; ++device1) {
// If it's not the same device, then validate P2P settings
if ((devices()[device0] != devices()[device1]) &&
static_cast<Device*>(devices()[device1])->settings().enableHwP2P_) {
Pal::GpuCompatibilityInfo comp_info = {};
// Can device 0 have access to device1?
static_cast<Device*>(devices()[device0])
->iDev()
->GetMultiGpuCompatibility(*static_cast<Device*>(devices()[device1])->iDev(),
&comp_info);
// Check P2P capability
if (comp_info.flags.peerTransfer) {
devices()[device0]->p2pDevices_.push_back(as_cl(devices()[device1]));
devices()[device1]->p2p_access_devices_.push_back(devices()[device0]);
}
}
}
}
}
return true;
}
@@ -1516,6 +1539,12 @@ pal::Memory* Device::createBuffer(amd::Memory& owner, bool directAccess) const {
params.owner_ = &owner;
params.gpu_ = static_cast<VirtualGPU*>(owner.getVirtualDevice());
params.svmBase_ = static_cast<Memory*>(owner.svmBase());
if (owner.P2PAccess()) {
params.svmBase_ = static_cast<Memory*>(owner.BaseP2PMemory());
if (params.svmBase_ != nullptr) {
type = Resource::P2PAccess;
}
}
// Create memory object
result = gpuMemory->create(type, &params);
@@ -1531,7 +1560,8 @@ pal::Memory* Device::createBuffer(amd::Memory& owner, bool directAccess) const {
pipeInit);
}
// If memory has direct access from host, then get CPU address
if (gpuMemory->isHostMemDirectAccess() && (type != Resource::ExternalPhysical)) {
if (gpuMemory->isHostMemDirectAccess() && (type != Resource::ExternalPhysical) &&
(type != Resource::P2PAccess)) {
void* address = gpuMemory->map(nullptr);
if (address != nullptr) {
// Copy saved memory
rocclr/runtime/device/pal/palmemory.cpp
+2 -1
Просмотреть файл
@@ -119,6 +119,7 @@ bool Memory::create(Resource::MemoryType memType, Resource::CreateParams* params
switch (memoryType()) {
case Resource::Pinned:
case Resource::ExternalPhysical:
case Resource::P2PAccess:
// Marks memory object for direct GPU access to the host memory
flags_ |= HostMemoryDirectAccess;
break;
@@ -903,7 +904,7 @@ void Memory::mgpuCacheWriteBack() {
amd::ScopedLock lock(owner()->lockMemoryOps());
// Attempt to allocate a staging buffer if don't have any
if (owner()->getHostMem() == nullptr) {
if (!owner()->P2PAccess() && (owner()->getHostMem() == nullptr)) {
if (nullptr != owner()->getSvmPtr()) {
owner()->commitSvmMemory();
owner()->setHostMem(owner()->getSvmPtr());
rocclr/runtime/device/pal/palresource.cpp
+69 -29
Просмотреть файл
@@ -152,6 +152,26 @@ GpuMemoryReference* GpuMemoryReference::Create(const Device& dev,
return memRef;
}
// ================================================================================================
GpuMemoryReference* GpuMemoryReference::Create(const Device& dev,
const Pal::PeerGpuMemoryOpenInfo& openInfo) {
Pal::Result result;
size_t gpuMemSize = dev.iDev()->GetPeerGpuMemorySize(openInfo, &result);
if (result != Pal::Result::Success) {
return nullptr;
}
GpuMemoryReference* memRef = new (gpuMemSize) GpuMemoryReference(dev);
if (memRef != nullptr) {
result = dev.iDev()->OpenPeerGpuMemory(openInfo, &memRef[1], &memRef->gpuMem_);
if (result != Pal::Result::Success) {
memRef->release();
return nullptr;
}
}
return memRef;
}
// ================================================================================================
GpuMemoryReference::GpuMemoryReference(const Device& dev)
: gpuMem_(nullptr), cpuAddress_(nullptr), device_(dev), gpu_(nullptr) {}
@@ -367,6 +387,7 @@ void Resource::memTypeToHeap(Pal::GpuMemoryCreateInfo* createInfo) {
createInfo->heapCount = 2;
createInfo->heaps[0] = Pal::GpuHeapLocal;
createInfo->heaps[1] = Pal::GpuHeapGartUswc;
createInfo->flags.peerWritable = dev().P2PAccessAllowed();
#ifdef ATI_OS_LINUX
// Note: SSG in Linux requires DGMA heap
if (dev().properties().gpuMemoryProperties.busAddressableMemSize > 0) {
@@ -391,6 +412,7 @@ void Resource::memTypeToHeap(Pal::GpuMemoryCreateInfo* createInfo) {
createInfo->heaps[0] = Pal::GpuHeapInvisible;
createInfo->heaps[1] = Pal::GpuHeapLocal;
createInfo->heaps[2] = Pal::GpuHeapGartUswc;
createInfo->flags.peerWritable = dev().P2PAccessAllowed();
break;
default:
createInfo->heaps[0] = Pal::GpuHeapLocal;
@@ -903,6 +925,20 @@ bool Resource::CreateInterop(CreateParams* params) {
return true;
}
// ================================================================================================
bool Resource::CreateP2PAccess(CreateParams* params) {
Pal::PeerGpuMemoryOpenInfo openInfo = {};
openInfo.pOriginalMem = params->svmBase_->iMem();
memRef_ = GpuMemoryReference::Create(dev(), openInfo);
if (nullptr == memRef_) {
return false;
}
desc_.cardMemory_ = false;
offset_ = params->svmBase_->offset();
return true;
}
// ================================================================================================
bool Resource::CreatePinned(CreateParams* params) {
PinnedParams* pinned = reinterpret_cast<PinnedParams*>(params);
@@ -1070,42 +1106,45 @@ bool Resource::create(MemoryType memType, CreateParams* params) {
if (dev().settings().disablePersistent_ && (memoryType() == Persistent)) {
desc_.type_ = RemoteUSWC;
}
switch (memoryType()) {
case OGLInterop:
case D3D9Interop:
case D3D10Interop:
case D3D11Interop:
return CreateInterop(params);
case P2PAccess:
return CreateP2PAccess(params);
case Pinned:
return CreatePinned(params);
case View: {
// Save the offset in the global heap
ViewParams* view = reinterpret_cast<ViewParams*>(params);
offset_ = view->offset_;
if ((memoryType() == OGLInterop) || (memoryType() == D3D9Interop) ||
(memoryType() == D3D10Interop) || (memoryType() == D3D11Interop)) {
return CreateInterop(params);
// Make sure parent was provided
if (nullptr != view->resource_) {
viewOwner_ = view->resource_;
offset_ += viewOwner_->offset();
if (viewOwner_->data() != nullptr) {
address_ = viewOwner_->data() + view->offset_;
mapCount_++;
}
memRef_ = viewOwner_->memRef_;
memRef_->retain();
desc_.cardMemory_ = viewOwner_->desc().cardMemory_;
} else {
desc_.type_ = Empty;
}
return true;
}
default:
break;
}
if (!desc_.buffer_) {
return CreateImage(params);
}
if (memoryType() == Pinned) {
return CreatePinned(params);
}
if (memoryType() == View) {
// Save the offset in the global heap
ViewParams* view = reinterpret_cast<ViewParams*>(params);
offset_ = view->offset_;
// Make sure parent was provided
if (nullptr != view->resource_) {
viewOwner_ = view->resource_;
offset_ += viewOwner_->offset();
if (viewOwner_->data() != nullptr) {
address_ = viewOwner_->data() + view->offset_;
mapCount_++;
}
memRef_ = viewOwner_->memRef_;
memRef_->retain();
desc_.cardMemory_ = viewOwner_->desc().cardMemory_;
} else {
desc_.type_ = Empty;
}
return true;
}
Pal::gpusize svmPtr = 0;
if ((nullptr != params) && (nullptr != params->owner_) &&
(nullptr != params->owner_->getSvmPtr())) {
@@ -1823,6 +1862,7 @@ bool MemorySubAllocator::CreateChunk(const Pal::IGpuMemory* reserved_va) {
createInfo.priority = Pal::GpuMemPriority::Normal;
createInfo.heapCount = 1;
createInfo.heaps[0] = Pal::GpuHeapInvisible;
createInfo.flags.peerWritable = device_->P2PAccessAllowed();
GpuMemoryReference* mem_ref = GpuMemoryReference::Create(*device_, createInfo);
if (mem_ref != nullptr) {
return InitAllocator(mem_ref);
rocclr/runtime/device/pal/palresource.hpp
+15 -5
Просмотреть файл
@@ -35,6 +35,8 @@ class GpuMemoryReference : public amd::ReferenceCountedObject {
static GpuMemoryReference* Create(const Device& dev, const Pal::ExternalImageOpenInfo& openInfo,
Pal::ImageCreateInfo* imgCreateInfo, Pal::IImage** image);
static GpuMemoryReference* Create(const Device& dev, const Pal::PeerGpuMemoryOpenInfo& openInfo);
//! Default constructor
GpuMemoryReference(const Device& dev);
@@ -53,10 +55,10 @@ class GpuMemoryReference : public amd::ReferenceCountedObject {
private:
//! Disable copy constructor
GpuMemoryReference(const GpuMemoryReference&);
GpuMemoryReference(const GpuMemoryReference&) = delete;
//! Disable operator=
GpuMemoryReference& operator=(const GpuMemoryReference&);
GpuMemoryReference& operator=(const GpuMemoryReference&) = delete;
};
static constexpr Pal::gpusize MaxGpuAlignment = 4 * Ki;
@@ -144,6 +146,7 @@ class Resource : public amd::HeapObject {
D3D9Interop, //!< resource is a D3D9 memory object
Scratch, //!< resource is scratch memory
Shader, //!< resource is a shader
P2PAccess //!< resource is a shared resource for P2P access
};
//! Resource map flags
@@ -416,6 +419,13 @@ class Resource : public amd::HeapObject {
bool CreateSvm(CreateParams* params, //!< special parameters for resource allocation
Pal::gpusize svmPtr);
/*! \brief Creates a PAL P2P object, associated with the resource
*
* \return True if we succesfully created a PAL P2P resource
*/
bool CreateP2PAccess(CreateParams* params //!< special parameters for resource allocation
);
uint elementSize_; //!< Size of a single element in bytes
private:
@@ -567,9 +577,9 @@ class ResourceCache : public amd::HeapObject {
amd::Monitor lockCacheOps_; //!< Lock to serialise cache access
size_t cacheSize_; //!< Current cache size in bytes
size_t lclCacheSize_; //!< Local memory stored in the cache
const size_t cacheSizeLimit_; //!< Cache size limit in bytes
size_t cacheSize_; //!< Current cache size in bytes
size_t lclCacheSize_; //!< Local memory stored in the cache
const size_t cacheSizeLimit_; //!< Cache size limit in bytes
//! PAL resource cache
std::list<std::pair<Resource::Descriptor*, GpuMemoryReference*> > resCache_;
rocclr/runtime/device/pal/palsettings.cpp
+7 -1
Просмотреть файл
@@ -29,7 +29,6 @@ struct ModifyMaxWorkload {
#endif
};
Settings::Settings() {
// Initialize the GPU device default settings
oclVersion_ = OpenCL12;
@@ -144,6 +143,7 @@ Settings::Settings() {
enableWave32Mode_ = false;
hsailExplicitXnack_ = false;
lcWavefrontSize64_ = true;
enableHwP2P_ = false;
}
bool Settings::create(const Pal::DeviceProperties& palProp,
@@ -213,6 +213,8 @@ bool Settings::create(const Pal::DeviceProperties& palProp,
lcWavefrontSize64_ = !enableWave32Mode_;
// Fall through to AI (gfx9) ...
case Pal::AsicRevision::Vega20:
// Enable HW P2P path for Vega20+. Runtime still relies on KMD/PAL for support
enableHwP2P_ = true;
case Pal::AsicRevision::Vega12:
case Pal::AsicRevision::Vega10:
case Pal::AsicRevision::Raven:
@@ -515,6 +517,10 @@ void Settings::override() {
resourceCacheSize_ = GPU_RESOURCE_CACHE_SIZE * Mi;
}
if (!flagIsDefault(GPU_ENABLE_HW_P2P)) {
enableHwP2P_ = GPU_ENABLE_HW_P2P;
}
if (!flagIsDefault(AMD_GPU_FORCE_SINGLE_FP_DENORM)) {
switch (AMD_GPU_FORCE_SINGLE_FP_DENORM) {
case 0:
rocclr/runtime/device/pal/palsettings.hpp
+2 -1
Просмотреть файл
@@ -61,7 +61,8 @@ class Settings : public device::Settings {
uint rgpSqttWaitIdle_ : 1; //!< Wait for idle after SQTT trace
uint rgpSqttForceDisable_ : 1; //!< Disables SQTT
uint splitSizeForWin7_ : 1; //!< DMA flush split size for Win 7
uint reserved_ : 11;
uint enableHwP2P_ : 1; //!< Forces HW P2P path for testing
uint reserved_ : 10;
};
uint value_;
};
rocclr/runtime/device/pal/palvirtual.cpp
+18 -18
Просмотреть файл
@@ -821,13 +821,12 @@ bool VirtualGPU::create(bool profiling, uint deviceQueueSize, uint rtCUs,
// Check if device has SDMA engines
if (dev().numDMAEngines() != 0 && !PAL_DISABLE_SDMA) {
uint sdma;
// If only 1 DMA engine is available then use that one
if ((dev().numDMAEngines() < 2) || ((idx & 0x1) && !dev().settings().svmFineGrainSystem_)) {
// If only 1 SDMA engine is available then use that one, otherwise it's a round-robin manner
if ((dev().numDMAEngines() < 2) || ((idx + 1) & 0x1)) {
sdma = 0;
} else {
sdma = 1;
}
queues_[SdmaEngine] = Queue::Create(
*this, Pal::QueueTypeDma, sdma, cmdAllocator_, amd::CommandQueue::RealTimeDisabled,
amd::CommandQueue::Priority::Normal, residency_limit, max_cmd_buffers);
@@ -843,7 +842,8 @@ bool VirtualGPU::create(bool profiling, uint deviceQueueSize, uint rtCUs,
}
}
} else {
Unimplemented();
LogError("Runtme couldn't find compute queues!");
return false;
}
if (!managedBuffer_.create(Resource::RemoteUSWC)) {
@@ -1695,22 +1695,22 @@ void VirtualGPU::submitCopyMemoryP2P(amd::CopyMemoryP2PCommand& cmd) {
profilingBegin(cmd);
Memory* srcDevMem = static_cast<pal::Memory*>(
cmd.source().getDeviceMemory(*cmd.source().getContext().devices()[0]));
Memory* dstDevMem = static_cast<pal::Memory*>(
cmd.destination().getDeviceMemory(*cmd.destination().getContext().devices()[0]));
// Get the device memory objects for the current device
Memory* srcDevMem = dev().getGpuMemory(&cmd.source());
Memory* dstDevMem = dev().getGpuMemory(&cmd.destination());
bool p2pAllowed = false;
#if 0
// Loop through all available P2P devices for the destination buffer
for (auto agent: dstDevMem->dev().p2pAgents()) {
// Find the device, which is matching the current
if (agent.handle == dev().getBackendDevice().handle) {
p2pAllowed = true;
break;
}
bool p2pAllowed = true;
// If any device object is null, then no HW P2P and runtime has to use staging
if (srcDevMem == nullptr) {
srcDevMem = static_cast<pal::Memory*>(
cmd.source().getDeviceMemory(*cmd.source().getContext().devices()[0]));
p2pAllowed = false;
} else if (dstDevMem == nullptr) {
dstDevMem = static_cast<pal::Memory*>(
cmd.destination().getDeviceMemory(*cmd.destination().getContext().devices()[0]));
p2pAllowed = false;
}
#endif
// Synchronize source and destination memory
device::Memory::SyncFlags syncFlags;
rocclr/runtime/device/rocm/rocdevice.cpp
+2 -1
Просмотреть файл
@@ -1013,7 +1013,7 @@ bool Device::populateOCLDeviceConstants() {
for (auto agent: gpu_agents_) {
if (agent.handle != _bkendDevice.handle) {
hsa_status_t err;
// Can current GPU have access to another GPU memory pool
// Can another GPU (agent) have access to the current GPU memory pool (gpuvm_segment_)?
hsa_amd_memory_pool_access_t access;
err = hsa_amd_agent_memory_pool_get_info(agent, gpuvm_segment_, HSA_AMD_AGENT_MEMORY_POOL_INFO_ACCESS, &access);
if (err != HSA_STATUS_SUCCESS) {
@@ -1023,6 +1023,7 @@ bool Device::populateOCLDeviceConstants() {
// Find accessible p2p agents - i.e != HSA_AMD_MEMORY_POOL_ACCESS_NEVER_ALLOWED
if (HSA_AMD_MEMORY_POOL_ACCESS_ALLOWED_BY_DEFAULT == access ||
HSA_AMD_MEMORY_POOL_ACCESS_DISALLOWED_BY_DEFAULT == access) {
// Agent can have access to the current gpuvm_segment_
p2p_agents_.push_back(agent);
}
}
rocclr/runtime/platform/command.cpp
+8 -1
Просмотреть файл
@@ -552,7 +552,14 @@ bool CopyMemoryP2PCommand::validateMemory() {
LogPrintfError("Can't allocate memory size - 0x%08X bytes!", memory2_->getSize());
return false;
}
if (devices[0]->P2PStage() != nullptr) {
bool p2pStaging = false;
// Validate P2P memories on the current device, if any of them is null, then it's p2p staging
if ((nullptr == memory1_->getDeviceMemory(queue()->device())) ||
(nullptr == memory2_->getDeviceMemory(queue()->device()))) {
p2pStaging = true;
}
if (devices[0]->P2PStage() != nullptr && p2pStaging) {
amd::ScopedLock lock(devices[0]->P2PStageOps());
// Make sure runtime allocates memory on every device
for (uint d = 0; d < devices[0]->GlbCtx().devices().size(); ++d) {
rocclr/runtime/platform/memory.cpp
+40 -16
Просмотреть файл
@@ -70,14 +70,14 @@ Memory::Memory(Context& context, Type type, Flags flags, size_t size, void* svmP
version_(0),
lastWriter_(NULL),
interopObj_(NULL),
isParent_(false),
vDev_(NULL),
forceSysMemAlloc_(false),
mapCount_(0),
svmHostAddress_(svmPtr),
svmPtrCommited_(flags & CL_MEM_SVM_FINE_GRAIN_BUFFER ? true : false),
canBeCached_(true),
lockMemoryOps_("Memory Ops Lock", true) {}
flagsEx_(0),
lockMemoryOps_("Memory Ops Lock", true) {
svmPtrCommited_ = (flags & CL_MEM_SVM_FINE_GRAIN_BUFFER) ? true : false;
canBeCached_ = true;
}
Memory::Memory(Memory& parent, Flags flags, size_t origin, size_t size, Type type)
: numDevices_(0),
@@ -93,14 +93,13 @@ Memory::Memory(Memory& parent, Flags flags, size_t origin, size_t size, Type typ
version_(parent.getVersion()),
lastWriter_(parent.getLastWriter()),
interopObj_(parent.getInteropObj()),
isParent_(false),
vDev_(NULL),
forceSysMemAlloc_(false),
mapCount_(0),
svmHostAddress_(parent.getSvmPtr()),
svmPtrCommited_(parent.isSvmPtrCommited()),
canBeCached_(true),
flagsEx_(0),
lockMemoryOps_("Memory Ops Lock", true) {
svmPtrCommited_ = parent.isSvmPtrCommited();
canBeCached_ = true;
parent_->retain();
parent_->isParent_ = true;
@@ -126,13 +125,31 @@ Memory::Memory(Memory& parent, Flags flags, size_t origin, size_t size, Type typ
}
}
uint32_t Memory::NumDevicesWithP2P() {
uint32_t devices = context_().devices().size();
if (devices == 1) {
// Add p2p devices for allocation
devices = context_().devices().size() + context_().devices()[0]->P2PAccessDevices().size();
if (devices > 1) {
p2pAccess_ = true;
}
}
return devices;
}
void Memory::initDeviceMemory() {
deviceMemories_ = reinterpret_cast<DeviceMemory*>(reinterpret_cast<char*>(this) + sizeof(Memory));
memset(deviceMemories_, 0, context_().devices().size() * sizeof(DeviceMemory));
memset(deviceMemories_, 0, NumDevicesWithP2P() * sizeof(DeviceMemory));
}
void* Memory::operator new(size_t size, const Context& context) {
return RuntimeObject::operator new(size + context.devices().size() * sizeof(DeviceMemory));
uint32_t devices = context.devices().size();
if (devices == 1) {
// Add p2p devices for allocation
devices = context.devices().size() + context.devices()[0]->P2PAccessDevices().size();
}
return RuntimeObject::operator new(size + devices * sizeof(DeviceMemory));
}
void Memory::operator delete(void* p) { RuntimeObject::operator delete(p); }
@@ -259,7 +276,14 @@ bool Memory::addDeviceMemory(const Device* dev) {
bool result = false;
AllocState create = AllocCreate;
AllocState init = AllocInit;
if (make_atomic(deviceAlloced_[dev]).compareAndSet(init, create)) {
// Check if runtime already allocated all available slots for device memory
if (numDevices() == NumDevicesWithP2P()) {
// Mark the allocation as an empty
deviceAlloced_[dev] = AllocInit;
return false;
}
device::Memory* dm = dev->createMemory(*this);
// Add the new memory allocation to the device map
@@ -267,7 +291,7 @@ bool Memory::addDeviceMemory(const Device* dev) {
deviceMemories_[numDevices_].ref_ = dev;
deviceMemories_[numDevices_].value_ = dm;
numDevices_++;
assert((numDevices() <= context_().devices().size()) && "Too many device objects");
assert((numDevices() <= NumDevicesWithP2P()) && "Too many device objects");
// Mark the allocation with the complete flag
deviceAlloced_[dev] = AllocComplete;
@@ -275,6 +299,7 @@ bool Memory::addDeviceMemory(const Device* dev) {
svmBase_ = dm;
}
} else {
LogError("Video memory allocation failed!");
// Mark the allocation as an empty
deviceAlloced_[dev] = AllocInit;
}
@@ -322,7 +347,6 @@ device::Memory* Memory::getDeviceMemory(const Device& dev, bool alloc) {
if ((NULL == dm) && alloc) {
if (!addDeviceMemory(&dev)) {
LogError("Video memory allocation failed!");
return NULL;
}
dm = deviceMemories_[numDevices() - 1].value_;
@@ -446,7 +470,7 @@ void Memory::uncommitSvmMemory() {
void Buffer::initDeviceMemory() {
deviceMemories_ = reinterpret_cast<DeviceMemory*>(reinterpret_cast<char*>(this) + sizeof(Buffer));
memset(deviceMemories_, 0, context_().devices().size() * sizeof(DeviceMemory));
memset(deviceMemories_, 0, NumDevicesWithP2P() * sizeof(DeviceMemory));
}
bool Buffer::create(void* initFrom, bool sysMemAlloc, bool skipAlloc) {
@@ -472,7 +496,7 @@ bool Buffer::validateRegion(const Coord3D& origin, const Coord3D& region) const
void Pipe::initDeviceMemory() {
deviceMemories_ = reinterpret_cast<DeviceMemory*>(reinterpret_cast<char*>(this) + sizeof(Pipe));
memset(deviceMemories_, 0, context_().devices().size() * sizeof(DeviceMemory));
memset(deviceMemories_, 0, NumDevicesWithP2P() * sizeof(DeviceMemory));
}
#define GETMIPDIM(dim, mip) (((dim >> mip) > 0) ? (dim >> mip) : 1)
@@ -631,7 +655,7 @@ void Image::initDimension() {
void Image::initDeviceMemory() {
deviceMemories_ = reinterpret_cast<DeviceMemory*>(reinterpret_cast<char*>(this) + sizeof(Image));
memset(deviceMemories_, 0, context_().devices().size() * sizeof(DeviceMemory));
memset(deviceMemories_, 0, NumDevicesWithP2P() * sizeof(DeviceMemory));
}
bool Image::create(void* initFrom) { return Memory::create(initFrom); }
rocclr/runtime/platform/memory.hpp
+46 -31
Просмотреть файл
@@ -24,7 +24,7 @@
namespace device {
class Memory;
class VirtualDevice;
}
} // namespace device
namespace amd {
@@ -42,7 +42,7 @@ struct BufferRect : public amd::EmbeddedObject {
const size_t* region, //!< Copy region
size_t bufferRowPitch, //!< Provided buffer's row pitch
size_t bufferSlicePitch //!< Provided buffer's slice pitch
);
);
//! Returns the plain offset for the (X, Y, Z) location
size_t offset(size_t x, //!< Coordinate in X dimension
@@ -120,6 +120,9 @@ class Memory : public amd::RuntimeObject {
typedef cl_mem_flags Flags;
typedef DeviceMap<const Device*, device::Memory*> DeviceMemory;
//! Returns the number of devices this memory object is associated, including P2P access
uint32_t NumDevicesWithP2P();
size_t numDevices_; //!< Number of devices
//! The device memory objects included in this memory
@@ -141,13 +144,19 @@ class Memory : public amd::RuntimeObject {
size_t version_; //!< Update count, used for coherency
const Device* lastWriter_; //!< Which device wrote most recently (NULL if host)
InteropObject* interopObj_; //!< Interop object
bool isParent_; //!< This object is a parent
device::VirtualDevice* vDev_; //!< Memory object belongs to a virtual device only
bool forceSysMemAlloc_; //!< Forces system memory allocation
std::atomic_uint mapCount_; //!< Keep track of number of mappings for a memory object
void* svmHostAddress_; //!< svm host address;
bool svmPtrCommited_; //!< svm host address committed flag;
bool canBeCached_; //!< flag to if the object can be cached;
union {
struct {
uint32_t isParent_ : 1; //!< This object is a parent
uint32_t forceSysMemAlloc_ : 1; //!< Forces system memory allocation
uint32_t svmPtrCommited_ : 1; //!< svm host address committed flag
uint32_t canBeCached_ : 1; //!< flag to if the object can be cached
uint32_t p2pAccess_ : 1; //!< Memory object allows P2P access
};
uint32_t flagsEx_;
};
private:
//! Disable default assignment operator
@@ -168,20 +177,20 @@ class Memory : public amd::RuntimeObject {
Flags flags, //!< Object's flags
size_t size, //!< Memory size
void* svmPtr = NULL //!< svm host memory address, NULL if no SVM mem object
);
);
Memory(Memory& parent, //!< Context object
Flags flags, //!< Object's flags
size_t offset, //!< Memory offset
size_t size, //!< Memory size
Type type = 0 //!< Memory type
);
);
//! Memory object destructor
virtual ~Memory();
//! Copies initialization data to the backing store
virtual void copyToBackingStore(void* initFrom //!< Pointer to the initialization memory
);
);
//! Initializes the device memory array
virtual void initDeviceMemory();
@@ -193,14 +202,14 @@ class Memory : public amd::RuntimeObject {
//! Placement new operator.
void* operator new(size_t size, //!< Original allocation size
const Context& context //!< Context this memory object is allocated in.
);
);
// Provide a "matching" placement delete operator.
void operator delete(void*, //!< Pointer to deallocate
const Context& context //!< Context this memory object is allocated in.
);
);
// and a regular delete operator to satisfy synthesized methods.
void operator delete(void* //!< Pointer to deallocate
);
);
//! Returns the memory lock object
amd::Monitor& lockMemoryOps() { return lockMemoryOps_; }
@@ -228,32 +237,32 @@ class Memory : public amd::RuntimeObject {
virtual Pipe* asPipe() { return NULL; }
//! Creates and initializes device (cache) memory for all devices
virtual bool create(void* initFrom = NULL, //!< Pointer to the initialization data
bool sysMemAlloc = false, //!< Allocate device memory in system memory
bool skipAlloc = false //!< Skip device memory allocation
);
virtual bool create(void* initFrom = NULL, //!< Pointer to the initialization data
bool sysMemAlloc = false, //!< Allocate device memory in system memory
bool skipAlloc = false //!< Skip device memory allocation
);
//! Allocates device (cache) memory for a specific device
bool addDeviceMemory(const Device* dev //!< Device object
);
);
//! Replaces device (cache) memory for a specific device
void replaceDeviceMemory(const Device* dev, //!< Device object
device::Memory* dm //!< New device memory object for replacement
);
);
//! Find the section for the given device. Return NULL if not found.
device::Memory* getDeviceMemory(const Device& dev, //!< Device object
bool alloc = true //!< Allocates memory
);
);
//! Allocate host memory (as required)
bool allocHostMemory(void* initFrom, //!< Host memory provided by the application
bool allocHostMem, //!< Force system memory allocation
bool forceCopy = false //!< Force system memory allocation
);
);
virtual void IpcCreate (size_t offset, size_t* mem_size, void* handle) const {
virtual void IpcCreate(size_t offset, size_t* mem_size, void* handle) const {
ShouldNotReachHere();
}
@@ -314,8 +323,14 @@ class Memory : public amd::RuntimeObject {
void uncommitSvmMemory();
void setCacheStatus(bool canBeCached) {
canBeCached_ = canBeCached;
} //!< set the memobject cached status;
bool canBeCached() const { return canBeCached_; } //!< get the memobject cached status;
} //!< set the memobject cached status
bool canBeCached() const { return canBeCached_; } //!< get the memobject cached status
//! Check if this objects allows P2P access
bool P2PAccess() const { return p2pAccess_; }
//! Returns the base device memory object for possible P2P access
device::Memory* BaseP2PMemory() const { return deviceMemories_[0].value_; }
device::Memory* svmBase() const { return svmBase_; } //!< Returns SVM base for MGPU case
};
@@ -339,10 +354,10 @@ class Buffer : public Memory {
Buffer(Memory& parent, Flags flags, size_t origin, size_t size)
: Memory(parent, flags, origin, size) {}
bool create(void* initFrom = NULL, //!< Pointer to the initialization data
bool sysMemAlloc = false, //!< Allocate device memory in system memory
bool skipAlloc = false //!< Skip device memory allocation
);
bool create(void* initFrom = NULL, //!< Pointer to the initialization data
bool sysMemAlloc = false, //!< Allocate device memory in system memory
bool skipAlloc = false //!< Skip device memory allocation
);
//! static_cast to Buffer with sanity check
virtual Buffer* asBuffer() { return this; }
@@ -463,7 +478,7 @@ class Image : public Memory {
//! Copies initialization data to the backing store
virtual void copyToBackingStore(void* initFrom //!< Pointer to the initialization memory
);
);
void initDimension();
@@ -482,7 +497,7 @@ class Image : public Memory {
size_t height, //!< Image height
size_t depth, //!< Image depth
size_t arraySize //!< Image array size
);
);
const Format& getImageFormat() const { return impl_.format_; }
@@ -512,7 +527,7 @@ class Image : public Memory {
device::VirtualDevice* vDev, //!< Virtual device object
uint baseMipLevel = 0, //!< Base mip level for a view
cl_mem_flags flags = 0 //!< Memory allocation flags
);
);
//! Returns the impl for this image.
Impl& getImpl() { return impl_; }
@@ -554,7 +569,7 @@ class Image : public Memory {
//! Creates and initializes device (cache) memory for all devices
bool create(void* initFrom = NULL //!< Pointer to the initialization data
);
);
};
//! SVM-related functionality.
rocclr/runtime/utils/flags.hpp
+8 -6
Просмотреть файл
@@ -168,17 +168,19 @@ release_on_stg(bool, PAL_DISABLE_SDMA, false, \
release(uint, PAL_RGP_DISP_COUNT, 50, \
"The number of dispatches for RGP capture with SQTT") \
release(bool, GPU_ENABLE_WAVE32_MODE, true, \
"Enables Wave32 compilation in HW if available") \
"Enables Wave32 compilation in HW if available") \
release(bool, GPU_ENABLE_LC, IS_LIGHTNING, \
"Enables LC path") \
release(bool, GPU_ENABLE_HW_P2P, false, \
"Enables HW P2P path") \
release(uint, GPU_MAX_COMMAND_BUFFERS, 8, \
"The maximum number of command buffers allocated per queue") \
release(cstring, HIP_VISIBLE_DEVICES, "", \
release(cstring, HIP_VISIBLE_DEVICES, "", \
"Only devices whose index is present in the sequence are visible to HIP") \
release(cstring, CUDA_VISIBLE_DEVICES, "", \
"Only devices whose index is present in the sequence are visible to HIP") \
release(cstring, CUDA_VISIBLE_DEVICES, "", \
"Only devices whose index is present in the sequence are visible to HIP") \
release(bool, GPU_ENABLE_WGP_MODE, true, \
"Enables WGP Mode in HW if available") \
release(bool, GPU_ENABLE_WGP_MODE, true, \
"Enables WGP Mode in HW if available") \
namespace amd {