AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SWDEV-491375 - Improve MemObjMap perf

Browse Source 
- Create bins each with its own map and lock. This would help cases
where the hash of a VA is differnet than ther one which falls in
different bin, and there is no lock contention
- Use STL shared mutexes, that way we can unique_lock for map updates
vs simple reads which can use shared_lock

Change-Id: I118818be65c6373700f5e511045babb6a398938a
This commit is contained in:
Saleel Kudchadker
2024-10-23 02:09:03 +00:00
parent 403f624bf8
commit e23ff0520b
2 changed files with 96 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rocclr/device/device.cpp
+62 -59
View File
@@ -305,18 +305,13 @@ Context* Device::glb_ctx_ = nullptr;
Monitor Device::p2p_stage_ops_(true);
Memory* Device::p2p_stage_ = nullptr;
Monitor MemObjMap::AllocatedLock_ ROCCLR_INIT_PRIORITY(101) ("Guards MemObjMap allocation list");
std::map<uintptr_t, amd::Memory*> MemObjMap::MemObjMap_ ROCCLR_INIT_PRIORITY(101);
std::map<uintptr_t, amd::Memory*> MemObjMap::VirtualMemObjMap_ ROCCLR_INIT_PRIORITY(101);
size_t MemObjMap::size() {
amd::ScopedLock lock(AllocatedLock_);
return MemObjMap_.size();
}
std::array<MemObjMap::Bin, MemObjMap::kNumBins> MemObjMap::MemObjBin_ ROCCLR_INIT_PRIORITY(101) = {} ;
std::array<MemObjMap::Bin, MemObjMap::kNumBins> MemObjMap::VirtualMemObjBin_ ROCCLR_INIT_PRIORITY(101) = {} ;
void MemObjMap::AddMemObj(const void* k, amd::Memory* v) {
amd::ScopedLock lock(AllocatedLock_);
auto rval = MemObjMap_.insert({ reinterpret_cast<uintptr_t>(k), v });
auto& bin = getMapBin(k, MemObjBin_);
std::unique_lock lock(bin.AllocatedLock_);
auto rval = bin.map_.insert({ reinterpret_cast<uintptr_t>(k), v });
if (!rval.second) {
DevLogPrintfError("Memobj map already has an entry for ptr: 0x%x",
reinterpret_cast<uintptr_t>(k));
@@ -324,17 +319,19 @@ void MemObjMap::AddMemObj(const void* k, amd::Memory* v) {
}
void MemObjMap::RemoveMemObj(const void* k) {
amd::ScopedLock lock(AllocatedLock_);
auto rval = MemObjMap_.erase(reinterpret_cast<uintptr_t>(k));
auto& bin = getMapBin(k, MemObjBin_);
std::unique_lock lock(bin.AllocatedLock_);
auto rval = bin.map_.erase(reinterpret_cast<uintptr_t>(k));
guarantee(rval == 1, "Memobj map does not have ptr: 0x%x",
reinterpret_cast<uintptr_t>(k));
}
amd::Memory* MemObjMap::FindMemObj(const void* k, size_t* offset) {
amd::ScopedLock lock(AllocatedLock_);
auto& bin = getMapBin(k, MemObjBin_);
std::shared_lock lock(bin.AllocatedLock_);
uintptr_t key = reinterpret_cast<uintptr_t>(k);
auto it = MemObjMap_.upper_bound(key);
if (it == MemObjMap_.begin()) {
auto it = bin.map_.upper_bound(key);
if (it == bin.map_.begin()) {
return nullptr;
}
@@ -350,9 +347,50 @@ amd::Memory* MemObjMap::FindMemObj(const void* k, size_t* offset) {
return nullptr;
}
}
void MemObjMap::UpdateAccess(amd::Device *peerDev) {
if (peerDev == nullptr) {
return;
}
// Provides access to all memory allocated on peerDev but
// hsa_amd_agents_allow_access was not called because there was no peer
for (auto& bin : MemObjBin_) {
std::shared_lock lock(bin.AllocatedLock_);
for (auto it : bin.map_) {
const std::vector<Device*>& devices = it.second->getContext().devices();
if (devices.size() == 1 && devices[0] == peerDev) {
device::Memory* devMem = it.second->getDeviceMemory(*devices[0]);
if (!devMem->getAllowedPeerAccess()) {
peerDev->deviceAllowAccess(reinterpret_cast<void*>(it.first));
devMem->setAllowedPeerAccess(true);
}
}
}
}
}
void MemObjMap::Purge(amd::Device* dev) {
assert(dev != nullptr);
for (auto& bin : MemObjBin_) {
std::unique_lock lock(bin.AllocatedLock_);
for (auto it = bin.map_.cbegin(); it != bin.map_.cend(); ) {
amd::Memory* memObj = it->second;
unsigned int flags = memObj->getMemFlags();
const std::vector<Device*>& devices = memObj->getContext().devices();
if (devices.size() == 1 && devices[0] == dev && !(flags & ROCCLR_MEM_INTERNAL_MEMORY)) {
memObj->release();
it = bin.map_.erase(it);
} else {
++it;
}
}
}
}
void MemObjMap::AddVirtualMemObj(const void* k, amd::Memory* v) {
amd::ScopedLock lock(AllocatedLock_);
auto rval = VirtualMemObjMap_.insert({ reinterpret_cast<uintptr_t>(k), v });
auto& bin = getMapBin(k, VirtualMemObjBin_);
std::unique_lock lock(bin.AllocatedLock_);
auto rval = bin.map_.insert({ reinterpret_cast<uintptr_t>(k), v });
if (!rval.second) {
DevLogPrintfError("Virtual Memobj map already has an entry for ptr: 0x%x",
reinterpret_cast<uintptr_t>(k));
@@ -360,17 +398,19 @@ void MemObjMap::AddVirtualMemObj(const void* k, amd::Memory* v) {
}
void MemObjMap::RemoveVirtualMemObj(const void* k) {
amd::ScopedLock lock(AllocatedLock_);
auto rval = VirtualMemObjMap_.erase(reinterpret_cast<uintptr_t>(k));
auto& bin = getMapBin(k, VirtualMemObjBin_);
std::unique_lock lock(bin.AllocatedLock_);
auto rval = bin.map_.erase(reinterpret_cast<uintptr_t>(k));
guarantee(rval == 1, "Virtual Memobj map does not have ptr: 0x%x",
reinterpret_cast<uintptr_t>(k));
}
amd::Memory* MemObjMap::FindVirtualMemObj(const void* k) {
amd::ScopedLock lock(AllocatedLock_);
auto& bin = getMapBin(k, VirtualMemObjBin_);
std::shared_lock lock(bin.AllocatedLock_);
uintptr_t key = reinterpret_cast<uintptr_t>(k);
auto it = VirtualMemObjMap_.upper_bound(key);
if (it == VirtualMemObjMap_.begin()) {
auto it = bin.map_.upper_bound(key);
if (it == bin.map_.begin()) {
return nullptr;
}
@@ -492,43 +532,6 @@ bool Device::DestroyVirtualBuffer(amd::Memory* vaddr_mem_obj) {
return true;
}
void MemObjMap::UpdateAccess(amd::Device *peerDev) {
if (peerDev == nullptr) {
return;
}
// Provides access to all memory allocated on peerDev but
// hsa_amd_agents_allow_access was not called because there was no peer
amd::ScopedLock lock(AllocatedLock_);
for (auto it : MemObjMap_) {
const std::vector<Device*>& devices = it.second->getContext().devices();
if (devices.size() == 1 && devices[0] == peerDev) {
device::Memory* devMem = it.second->getDeviceMemory(*devices[0]);
if (!devMem->getAllowedPeerAccess()) {
peerDev->deviceAllowAccess(reinterpret_cast<void*>(it.first));
devMem->setAllowedPeerAccess(true);
}
}
}
}
void MemObjMap::Purge(amd::Device* dev) {
assert(dev != nullptr);
amd::ScopedLock lock(AllocatedLock_);
for (auto it = MemObjMap_.cbegin(); it != MemObjMap_.cend(); ) {
amd::Memory* memObj = it->second;
unsigned int flags = memObj->getMemFlags();
const std::vector<Device*>& devices = memObj->getContext().devices();
if (devices.size() == 1 && devices[0] == dev && !(flags & ROCCLR_MEM_INTERNAL_MEMORY)) {
memObj->release();
it = MemObjMap_.erase(it);
} else {
++it;
}
}
}
Device::BlitProgram::~BlitProgram() {
if (program_ != nullptr) {
program_->release();
rocclr/device/device.hpp
+34 -19
View File
@@ -57,6 +57,8 @@
#include <set>
#include <unordered_set>
#include <utility>
#include <shared_mutex>
#include <array>
namespace amd {
class Command;
@@ -1360,27 +1362,40 @@ namespace amd {
//! MemoryObject map lookup class
class MemObjMap : public AllStatic {
public:
static size_t size(); //!< obtain the size of the container
static void AddMemObj(const void* k,
amd::Memory* v); //!< add the host mem pointer and buffer in the container
static void RemoveMemObj(const void* k); //!< Remove an entry of mem object from the container
static amd::Memory* FindMemObj(
const void* k, //!< find the mem object based on the input pointer
size_t* offset = nullptr); //!< Offset in the memory location
static void UpdateAccess(amd::Device *peerDev);
static void Purge(amd::Device* dev); //!< Purge all user allocated memories on the given device
//!< add the host mem pointer and buffer in the container
static void AddMemObj(const void* k, amd::Memory* v);
static void AddVirtualMemObj(const void* k,
amd::Memory* v); //!< Same as AddMemObj but for virtual addressing
static void RemoveVirtualMemObj(const void* k); //!< Same as RemoveMemObj but for virtual addressing
static amd::Memory* FindVirtualMemObj(
const void* k); //!< Same as FindMemObj but for virtual addressing
//!< Remove an entry of mem object from the container
static void RemoveMemObj(const void* k);
//!< Find the mem object based on the input pointer, outputs the offset
static amd::Memory* FindMemObj( const void* k, size_t* offset = nullptr);
static void UpdateAccess(amd::Device *peerDev);
//!< Purge all user allocated memories on the given device
static void Purge(amd::Device* dev);
//!< Same as AddMemObj but for virtual addressing
static void AddVirtualMemObj(const void* k, amd::Memory* v);
//!< Same as RemoveMemObj but for virtual addressing
static void RemoveVirtualMemObj(const void* k);
//!< Same as FindMemObj but for virtual addressing
static amd::Memory* FindVirtualMemObj(const void* k);
struct Bin {
std::shared_mutex AllocatedLock_;
std::map<uintptr_t, amd::Memory*> map_;
};
static constexpr size_t kNumBins = 16;
private:
static std::map<uintptr_t, amd::Memory*>
MemObjMap_; //!< the mem object<->hostptr information container
static std::map<uintptr_t, amd::Memory*>
VirtualMemObjMap_; //!< the virtual mem object<->hostptr information container
static amd::Monitor AllocatedLock_; //!< amd monitor locker
//!< the mem object<->hostptr information container
static std::array<Bin, kNumBins> MemObjBin_;
//!< the virtual mem object<->hostptr information container
static std::array<Bin, kNumBins> VirtualMemObjBin_;
static Bin& getMapBin(const void* k, std::array<Bin, kNumBins>& bin) {
size_t index = std::hash<const void*>{}(k) % kNumBins;
return bin.at(index);
}
};
/// @brief Instruction Set Architecture properties.