AI/rocm-systems
2
0
Fork 0
Código Incidencias Pull Requests Acciones Paquetes Proyectos Lanzamientos Wiki Actividad

SWDEV-569319 Replace ScopedAcquire with stdcpp wrappers (#2146)

Explorar el Código 
* SWDEV-569319 Replace ScopedAcquire with stdcpp wrappers

* Remove KernelMutex and KernelSharedMutex abstractions with std::mutex and std::shared_mutex

* Replaced unique_locks with lock_guards

* More changes

* Replace new and deletes with smart pointers

* Replaced some more with shared ptrs

* Replacements with smart pointers - pt 2

* missed change
Este commit está contenido en:
pghoshamd
2026-01-06 10:59:34 -05:00
cometido por GitHub
padre e005f8487b
commit 637b0d71f0
Se han modificado 34 ficheros con 319 adiciones y 452 borrados
Descargar archivo de parche Descargar archivo de diferencias Expandir todos los archivos Contraer todos los archivos
projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aie_agent.cpp
+11 -11
Ver fichero
@@ -66,7 +66,6 @@ AieAgent::AieAgent(uint32_t node, const HsaNodeProperties& node_props)
}
AieAgent::~AieAgent() {
std::for_each(regions_.begin(), regions_.end(), DeleteObject());
regions_.clear();
}
@@ -75,8 +74,8 @@ hsa_status_t AieAgent::VisitRegion(bool include_peer,
void *data),
void *data) const {
AMD::callback_t<decltype(callback)> call(callback);
for (const auto r : regions_) {
hsa_region_t region_handle(core::MemoryRegion::Convert(r));
for (const auto& r : regions_) {
hsa_region_t region_handle(core::MemoryRegion::Convert(r.get()));
hsa_status_t err = call(region_handle, data);
if (err != HSA_STATUS_SUCCESS) {
return err;
@@ -321,24 +320,25 @@ void AieAgent::InitRegionList() {
/// explicit sync operations.
regions_.reserve(3);
regions_.push_back(
new MemoryRegion(false, true, false, false, true, this, sys_mem_props));
std::make_shared<MemoryRegion>(false, true, false, false, true, this, sys_mem_props));
regions_.push_back(
new MemoryRegion(false, false, false, false, true, this, dev_mem_props));
regions_.push_back(new MemoryRegion(false, false, false, false, true, this,
other_mem_props));
std::make_shared<MemoryRegion>(false, false, false, false, true, this, dev_mem_props));
regions_.push_back(
std::make_shared<MemoryRegion>(false, false, false, false, true, this, other_mem_props));
}
void AieAgent::InitAllocators() {
for (const auto *region : regions()) {
for (const auto& region : regions()) {
const MemoryRegion *amd_mem_region(
static_cast<const MemoryRegion *>(region));
static_cast<const MemoryRegion *>(region.get()));
if (amd_mem_region->kernarg()) {
const core::MemoryRegion* region_ptr = region.get();
system_allocator_ =
[region](size_t size, size_t align,
[region_ptr](size_t size, size_t align,
core::MemoryRegion::AllocateFlags alloc_flags) -> void * {
void *mem(nullptr);
return (core::Runtime::runtime_singleton_->AllocateMemory(
region, size, alloc_flags, &mem) == HSA_STATUS_SUCCESS)
region_ptr, size, alloc_flags, &mem) == HSA_STATUS_SUCCESS)
? mem
: nullptr;
};
projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp
+11 -11
Ver fichero
@@ -165,8 +165,8 @@ AqlQueue::AqlQueue(core::SharedQueue* shared_queue, GpuAgent* agent, size_t req_
// Set group and private memory apertures in amd_queue_.
auto& regions = agent->regions();
for (auto region : regions) {
const MemoryRegion* amdregion = static_cast<const AMD::MemoryRegion*>(region);
for (const auto& region : regions) {
const MemoryRegion* amdregion = static_cast<const AMD::MemoryRegion*>(region.get());
uint64_t base = amdregion->GetBaseAddress();
if (amdregion->IsLDS()) {
@@ -217,7 +217,7 @@ AqlQueue::AqlQueue(core::SharedQueue* shared_queue, GpuAgent* agent, size_t req_
}
MAKE_NAMED_SCOPE_GUARD(EventGuard, [&]() {
ScopedAcquire<KernelMutex> _lock(&queue_lock());
std::lock_guard<std::mutex> _lock(queue_lock());
queue_count()--;
if (queue_count() == 0) {
core::InterruptSignal::DestroyEvent(queue_event());
@@ -232,7 +232,7 @@ AqlQueue::AqlQueue(core::SharedQueue* shared_queue, GpuAgent* agent, size_t req_
});
if (core::g_use_interrupt_wait) {
ScopedAcquire<KernelMutex> _lock(&queue_lock());
std::lock_guard<std::mutex> _lock(queue_lock());
queue_count()++;
if (queue_event() == nullptr) {
assert(queue_count() == 1 && "Inconsistency in queue event reference counting found.\n");
@@ -387,7 +387,7 @@ AqlQueue::~AqlQueue() {
FreeQueueMemory();
if (core::g_use_interrupt_wait) {
ScopedAcquire<KernelMutex> lock(&queue_lock());
std::lock_guard<std::mutex> lock(queue_lock());
queue_count()--;
if (queue_count() == 0) {
core::InterruptSignal::DestroyEvent(queue_event());
@@ -777,7 +777,7 @@ void AqlQueue::AsyncReclaimMainScratch() {
tool::notify_event_scratch_async_reclaim_start(public_handle(),
HSA_AMD_EVENT_SCRATCH_ALLOC_FLAG_NONE);
ScopedAcquire<KernelMutex> lock(&scratch_lock_);
std::lock_guard<std::mutex> lock(scratch_lock_);
// Unmap the queue. CP will check amd_queue_ fields on re-map
Suspend();
@@ -849,7 +849,7 @@ void AqlQueue::AsyncReclaimAltScratch() {
tool::notify_event_scratch_async_reclaim_start(public_handle(),
HSA_AMD_EVENT_SCRATCH_ALLOC_FLAG_ALT);
ScopedAcquire<KernelMutex> lock(&scratch_lock_);
std::lock_guard<std::mutex> lock(scratch_lock_);
// Unmap the queue. CP will check amd_queue_ fields on re-map
Suspend();
@@ -1014,7 +1014,7 @@ void AqlQueue::HandleInsufficientScratch(hsa_signal_value_t& error_code,
const uint64_t device_size = size_per_thread * lanes_per_wave * device_slots;
const uint64_t dispatch_size = size_per_thread * lanes_per_wave * dispatch_slots;
ScopedAcquire<KernelMutex> lock(&scratch_lock_);
std::lock_guard<std::mutex> lock(scratch_lock_);
// scratch.use_alt_limit will be 0 if alt scratch is not supported or disabled
if (dispatch_size < scratch.use_alt_limit && dispatch_slots < device_slots) {
@@ -1393,7 +1393,7 @@ hsa_status_t AqlQueue::SetCUMasking(uint32_t num_cu_mask_count, const uint32_t*
if ((mask.size() == mask_dwords) && (tail_mask != 0)) mask[mask_dwords - 1] &= tail_mask;
// Apply mask if non-default or not queue initialization.
ScopedAcquire<KernelMutex> lock(&mask_lock_);
std::lock_guard<std::mutex> lock(mask_lock_);
if ((!cu_mask_.empty()) || (num_cu_mask_count != 0) || (!global_mask.empty())) {
// Devices with WGPs must conform to even-indexed contiguous pairwise CU enablement.
@@ -1414,7 +1414,7 @@ hsa_status_t AqlQueue::SetCUMasking(uint32_t num_cu_mask_count, const uint32_t*
}
hsa_status_t AqlQueue::GetCUMasking(uint32_t num_cu_mask_count, uint32_t* cu_mask) {
ScopedAcquire<KernelMutex> lock(&mask_lock_);
std::lock_guard<std::mutex> lock(mask_lock_);
assert(!cu_mask_.empty() && "No current cu_mask!");
uint32_t user_dword_count = num_cu_mask_count / 32;
@@ -1440,7 +1440,7 @@ void AqlQueue::SetProfiling(bool enabled) {
void AqlQueue::ExecutePM4(uint32_t* cmd_data, size_t cmd_size_b, hsa_fence_scope_t acquireFence,
hsa_fence_scope_t releaseFence, hsa_signal_t* in_signal) {
// pm4_ib_buf_ is a shared resource, so mutually exclude here.
ScopedAcquire<KernelMutex> lock(&pm4_ib_mutex_);
std::lock_guard<std::mutex> lock(pm4_ib_mutex_);
// Obtain reference to any container queue.
core::Queue* queue = core::Queue::Convert(public_handle());
projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp
+2 -2
Ver fichero
@@ -293,7 +293,7 @@ static bool DepSignalCompleteHandler(hsa_signal_value_t signal_value, void *arg
template <bool useGCR>
hsa_status_t BlitSdma<useGCR>::SubmitBlockingCommand(const void* cmd, size_t cmd_size,
uint64_t size) {
ScopedAcquire<KernelMutex> lock(&lock_);
std::unique_lock<std::mutex> lock(lock_);
// Alternate between completion signals
// Using two allows overlapping command writing and copies
@@ -310,7 +310,7 @@ hsa_status_t BlitSdma<useGCR>::SubmitBlockingCommand(const void* cmd, size_t cmd
// Mark signal as in use, guard against exception leaving the signal in an unusable state.
completionSignal->StoreRelaxed(2);
MAKE_SCOPE_GUARD([&]() { completionSignal->StoreRelaxed(0); });
lock.Release();
lock.unlock();
std::vector<core::Signal*> gang_signals(0);
projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_cpu_agent.cpp
+7 -8
Ver fichero
@@ -64,7 +64,6 @@ CpuAgent::CpuAgent(HSAuint32 node, const HsaNodeProperties& node_props,
}
CpuAgent::~CpuAgent() {
std::for_each(regions_.begin(), regions_.end(), DeleteObject());
regions_.clear();
}
@@ -87,17 +86,17 @@ void CpuAgent::InitRegionList() {
if (system_prop != mem_props.end()) system_props = *system_prop;
// Fine-Grain Memory
regions_.push_back(new MemoryRegion(true, false, is_apu_node, false, true, this, system_props));
regions_.push_back(std::make_shared<MemoryRegion>(true, false, is_apu_node, false, true, this, system_props));
// Ext-Fine-Grain Memory
regions_.push_back(new MemoryRegion(false, false, is_apu_node, true, true, this, system_props));
regions_.push_back(std::make_shared<MemoryRegion>(false, false, is_apu_node, true, true, this, system_props));
// Kernargs
regions_.push_back(new MemoryRegion(true, true, is_apu_node, false, true, this, system_props));
regions_.push_back(std::make_shared<MemoryRegion>(true, true, is_apu_node, false, true, this, system_props));
if (!is_apu_node) {
// Coarse Grain
regions_.push_back(new MemoryRegion(false, false, is_apu_node, false, true, this, system_props));
regions_.push_back(std::make_shared<MemoryRegion>(false, false, is_apu_node, false, true, this, system_props));
}
}
}
@@ -150,12 +149,12 @@ hsa_status_t CpuAgent::VisitRegion(bool include_peer,
}
hsa_status_t CpuAgent::VisitRegion(
const std::vector<const core::MemoryRegion*>& regions,
const std::vector<std::shared_ptr<const core::MemoryRegion>>& regions,
hsa_status_t (*callback)(hsa_region_t region, void* data),
void* data) const {
for (const core::MemoryRegion* region : regions) {
for (const std::shared_ptr<const rocr::core::MemoryRegion>& region : regions) {
if (!region->user_visible()) continue;
hsa_region_t region_handle = core::MemoryRegion::Convert(region);
hsa_region_t region_handle = core::MemoryRegion::Convert(region.get());
hsa_status_t status = callback(region_handle, data);
if (status != HSA_STATUS_SUCCESS) {
return status;
projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp
+42 -41
Ver fichero
@@ -112,7 +112,9 @@ GpuAgent::GpuAgent(HSAuint32 node, const HsaNodeProperties& node_props, bool xna
scratch_limit_async_threshold_(0),
scratch_cache_(
[this](void* base, size_t size, bool large) { ReleaseScratch(base, size, large); }),
trap_handler_tma_region_(NULL),
trap_handler_tma_region_(nullptr, [this](void* ptr){
if (ptr && this->finegrain_allocator_) this->finegrain_deallocator()(ptr);
}),
rec_sdma_eng_override_(false),
pcs_hosttrap_data_(),
pcs_stochastic_data_(),
@@ -246,7 +248,6 @@ GpuAgent::GpuAgent(HSAuint32 node, const HsaNodeProperties& node_props, bool xna
GpuAgent::~GpuAgent() {
for (auto& blit : blits_) blit.reset();
std::for_each(regions_.begin(), regions_.end(), DeleteObject());
regions_.clear();
}
@@ -454,22 +455,20 @@ void GpuAgent::InitRegionList() {
memory_max_frequency_ = mem_props[mem_idx].MemoryClockMax;
case HSA_HEAPTYPE_GPU_LDS:
case HSA_HEAPTYPE_GPU_SCRATCH: {
MemoryRegion* region =
new MemoryRegion(false, false, false, false, true, this, mem_props[mem_idx]);
std::shared_ptr<MemoryRegion> region = std::make_shared<MemoryRegion>(false, false, false, false, true, this, mem_props[mem_idx]);
regions_.push_back(region);
if (region->IsLocalMemory()) {
// Extended Fine-Grain memory
if (!(isa_->GetMajorVersion() == 12 && isa_->GetMinorVersion() == 0))
regions_.push_back(
new MemoryRegion(false, false, false, true, true, this, mem_props[mem_idx]));
std::make_shared<MemoryRegion>(false, false, false, true, true, this, mem_props[mem_idx]));
// Expose VRAM as uncached/fine grain over PCIe (if enabled) or XGMI.
bool user_visible = (properties_.HiveID != 0) ||
core::Runtime::runtime_singleton_->flag().fine_grain_pcie();
regions_.push_back(new MemoryRegion(true, false, false, false, user_visible, this,
regions_.push_back(std::make_shared<MemoryRegion>(true, false, false, false, user_visible, this,
mem_props[mem_idx]));
}
break;
@@ -561,7 +560,7 @@ void GpuAgent::ReserveScratch()
size_t available;
hsa_status_t err = driver().AvailableMemory(node_id(), &available);
assert(err == HSA_STATUS_SUCCESS && "AvailableMemory failed");
ScopedAcquire<KernelMutex> lock(&scratch_lock_);
std::lock_guard<std::mutex> lock(scratch_lock_);
if (!scratch_cache_.reserved_bytes() && reserved_sz && available > 8 * reserved_sz) {
HSAuint64 alt_va;
void* reserved_base = scratch_pool_.alloc(reserved_sz);
@@ -676,20 +675,20 @@ hsa_status_t GpuAgent::VisitRegion(bool include_peer,
}
hsa_status_t GpuAgent::VisitRegion(
const std::vector<const core::MemoryRegion*>& regions,
const std::vector<std::shared_ptr<const core::MemoryRegion>>& regions,
hsa_status_t (*callback)(hsa_region_t region, void* data),
void* data) const {
AMD::callback_t<decltype(callback)> call(callback);
for (const core::MemoryRegion* region : regions) {
for (const auto& region : regions) {
if (!region->user_visible()) continue;
const AMD::MemoryRegion* amd_region =
reinterpret_cast<const AMD::MemoryRegion*>(region);
reinterpret_cast<const AMD::MemoryRegion*>(region.get());
// Only expose system, local, and LDS memory.
if (amd_region->IsSystem() || amd_region->IsLocalMemory() ||
amd_region->IsLDS()) {
hsa_region_t region_handle = core::MemoryRegion::Convert(region);
hsa_region_t region_handle = core::MemoryRegion::Convert(region.get());
hsa_status_t status = call(region_handle, data);
if (status != HSA_STATUS_SUCCESS) {
return status;
@@ -910,7 +909,7 @@ void GpuAgent::InitGWS() {
}
void GpuAgent::GWSRelease() {
ScopedAcquire<KernelMutex> lock(&gws_queue_.lock_);
std::lock_guard<std::mutex> lock(gws_queue_.lock_);
gws_queue_.ref_ct_--;
if (gws_queue_.ref_ct_ != 0) return;
InitGWS();
@@ -968,22 +967,22 @@ hsa_status_t GpuAgent::DmaCopy(void* dst, const void* src, size_t size) {
}
void GpuAgent::SetCopyRequestRefCount(bool set) {
ScopedAcquire<KernelMutex> lock(&blit_lock_);
std::unique_lock<std::mutex> lock(blit_lock_);
while (pending_copy_stat_check_ref_) {
blit_lock_.Release();
lock.unlock();
os::YieldThread();
blit_lock_.Acquire();
lock.lock();
}
if (!set && pending_copy_req_ref_) pending_copy_req_ref_--;
else pending_copy_req_ref_++;
}
void GpuAgent::SetCopyStatusCheckRefCount(bool set) {
ScopedAcquire<KernelMutex> lock(&blit_lock_);
std::unique_lock<std::mutex> lock(blit_lock_);
while (pending_copy_req_ref_) {
blit_lock_.Release();
lock.unlock();
os::YieldThread();
blit_lock_.Acquire();
lock.lock();
}
if (!set && pending_copy_stat_check_ref_) pending_copy_stat_check_ref_--;
else pending_copy_stat_check_ref_++;
@@ -1059,7 +1058,7 @@ hsa_status_t GpuAgent::DmaCopy(void* dst, core::Agent& dst_agent,
std::min(gang_factor, properties_.NumSdmaXgmiEngines);
}
ScopedAcquire<KernelMutex> lock(&sdma_gang_lock_);
std::lock_guard<std::mutex> lock(sdma_gang_lock_);
// Manage internal gang signals
std::vector<core::Signal*> gang_signals;
if (gang_factor > 1) {
@@ -1642,7 +1641,7 @@ hsa_status_t GpuAgent::GetInfo(hsa_agent_info_t attribute, void* value) const {
if (status != HSA_STATUS_SUCCESS) return HSA_STATUS_ERROR_INVALID_ARGUMENT;
for (auto r : regions()) availableBytes += ((AMD::MemoryRegion*)r)->GetCacheSize();
for (const auto& r : regions()) availableBytes += ((AMD::MemoryRegion*)(r.get()))->GetCacheSize();
availableBytes += scratch_cache_.free_bytes() - scratch_cache_.reserved_bytes();
@@ -1730,7 +1729,7 @@ hsa_status_t GpuAgent::QueueCreate(size_t size, hsa_queue_type32_t queue_type, u
core::Queue** queue) {
// Handle GWS queues.
if (queue_type == HSA_QUEUE_TYPE_COOPERATIVE) {
ScopedAcquire<KernelMutex> lock(&gws_queue_.lock_);
std::lock_guard<std::mutex> lock(gws_queue_.lock_);
auto ret = (*gws_queue_.queue_).get();
if (ret != nullptr) {
gws_queue_.ref_ct_++;
@@ -1876,7 +1875,7 @@ void GpuAgent::AcquireQueueMainScratch(ScratchInfo& scratch) {
*/
bool large;
ScopedAcquire<KernelMutex> lock(&scratch_lock_);
std::lock_guard<std::mutex> lock(scratch_lock_);
const size_t small_limit = scratch_pool_.size() >> 3;
bool use_reclaim = true;
@@ -2035,7 +2034,7 @@ void GpuAgent::AcquireQueueAltScratch(ScratchInfo& scratch) {
uint64_t size_per_wave = AlignUp(scratch.alt_size_per_thread * properties_.WaveFrontSize, 1024);
if (size_per_wave > MAX_WAVE_SCRATCH) return;
ScopedAcquire<KernelMutex> lock(&scratch_lock_);
std::lock_guard<std::mutex> lock(scratch_lock_);
// Ensure mapping will be in whole pages.
scratch.alt_size = AlignUp(scratch.alt_size, 4096);
@@ -2176,7 +2175,7 @@ uint64_t GpuAgent::TranslateTime(uint64_t tick) {
// Limit errors due to relative frequency drift to ~0.5us. Sync clocks at 16Hz.
const int64_t max_extrapolation = core::Runtime::runtime_singleton_->sys_clock_freq() >> 4;
ScopedAcquire<KernelMutex> lock(&t1_lock_);
std::lock_guard<std::mutex> lock(t1_lock_);
// Limit errors due to correlated pair certainty to ~0.5us.
// extrapolated time < (0.5us / half clock read certainty) * delay between clock measures
// clock read certainty is <4us.
@@ -2261,26 +2260,27 @@ hsa_status_t GpuAgent::UpdateTrapHandlerWithPCS(pcs_sampling_data_t* pcs_hosttra
((uint64_t*)tma_region_host)[1] = (uint64_t)pcs_stochastic_buffers;
if (!trap_handler_tma_region_) {
trap_handler_tma_region_ = (uint64_t*)finegrain_allocator()(2 * sizeof(uint64_t), 0);
if (trap_handler_tma_region_ == nullptr) return HSA_STATUS_ERROR_OUT_OF_RESOURCES;
void* mem = (uint64_t*)finegrain_allocator()(2 * sizeof(uint64_t), 0);
if (!mem) return HSA_STATUS_ERROR_OUT_OF_RESOURCES;
trap_handler_tma_region_.reset(mem);
// NearestCpuAgent owns pool returned system_allocator()
auto cpuAgent = GetNearestCpuAgent()->public_handle();
hsa_status_t ret =
AMD::hsa_amd_agents_allow_access(1, &cpuAgent, NULL, trap_handler_tma_region_);
AMD::hsa_amd_agents_allow_access(1, &cpuAgent, NULL, trap_handler_tma_region_.get());
assert(ret == HSA_STATUS_SUCCESS);
}
/* On non-large BAR systems, we may not be able to access device memory, so do a DmaCopy */
if (DmaCopy(trap_handler_tma_region_, tma_region_host, 2 * sizeof(uint64_t)) != HSA_STATUS_SUCCESS)
if (DmaCopy(trap_handler_tma_region_.get(), tma_region_host, 2 * sizeof(uint64_t)) != HSA_STATUS_SUCCESS)
return HSA_STATUS_ERROR;
tma_size = 2 * sizeof(uint64_t);
tma_addr = trap_handler_tma_region_;
tma_addr = trap_handler_tma_region_.get();
} else if (trap_handler_tma_region_) {
finegrain_deallocator()(trap_handler_tma_region_);
trap_handler_tma_region_ = NULL;
trap_handler_tma_region_.reset(nullptr);
}
// Bind the trap handler to this node.
@@ -2398,7 +2398,7 @@ lazy_ptr<core::Blit>& GpuAgent::GetXgmiBlit(const core::Agent& dst_agent) {
uint32_t xgmi_engine_cnt = properties_.NumSdmaXgmiEngines;
assert((xgmi_engine_cnt > 0) && ("Illegal condition, should not happen"));
ScopedAcquire<KernelMutex> lock(&xgmi_peer_list_lock_);
std::lock_guard<std::mutex> lock(xgmi_peer_list_lock_);
for (uint32_t idx = 0; idx < xgmi_peer_list_.size(); idx++) {
uint64_t dst_handle = dst_agent.public_handle().handle;
@@ -2490,19 +2490,20 @@ lazy_ptr<core::Blit>& GpuAgent::GetBlitObject(const core::Agent& dst_agent,
void GpuAgent::Trim() {
Agent::Trim();
AsyncReclaimScratchQueues();
ScopedAcquire<KernelMutex> lock(&scratch_lock_);
std::lock_guard<std::mutex> lock(scratch_lock_);
scratch_cache_.trim(false);
}
void GpuAgent::InitAllocators() {
for (auto pool : GetNearestCpuAgent()->regions()) {
for (const auto& pool : GetNearestCpuAgent()->regions()) {
if (pool->kernarg()) {
system_allocator_ = [pool](size_t size, size_t alignment,
const core::MemoryRegion* pool_ptr = pool.get();
system_allocator_ = [pool_ptr](size_t size, size_t alignment,
MemoryRegion::AllocateFlags alloc_flags) -> void* {
assert(alignment <= 4096);
void* ptr = nullptr;
return (HSA_STATUS_SUCCESS ==
core::Runtime::runtime_singleton_->AllocateMemory(pool, size, alloc_flags, &ptr))
core::Runtime::runtime_singleton_->AllocateMemory(pool_ptr, size, alloc_flags, &ptr))
? ptr
: nullptr;
};
@@ -2513,14 +2514,14 @@ void GpuAgent::InitAllocators() {
assert(system_allocator_ && "Nearest NUMA node did not have a kernarg pool.");
// Setup this GPU's fine-grain and coarse-grain allocators.
for (auto region : regions()) {
const AMD::MemoryRegion* amd_region = static_cast<const AMD::MemoryRegion*>(region);
for (const auto& region : regions()) {
const AMD::MemoryRegion* amd_region = static_cast<const AMD::MemoryRegion*>(region.get());
auto region_allocator = [region](size_t size,
auto region_allocator = [amd_region](size_t size,
MemoryRegion::AllocateFlags alloc_flags) -> void* {
void* ptr = nullptr;
return (HSA_STATUS_SUCCESS ==
core::Runtime::runtime_singleton_->AllocateMemory(region, size, alloc_flags, &ptr))
core::Runtime::runtime_singleton_->AllocateMemory(amd_region, size, alloc_flags, &ptr))
? ptr
: nullptr;
};
projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_loader_context.cpp
+5 -5
Ver fichero
@@ -283,18 +283,18 @@ const core::MemoryRegion* RegionMemory::AgentLocal(hsa_agent_t agent, bool is_co
assert(amd_agent->device_type() == core::Agent::kAmdGpuDevice && "Invalid agent type.");
auto agent_local_region =
std::find_if(amd_agent->regions().begin(), amd_agent->regions().end(),
[&](const core::MemoryRegion* region) {
const AMD::MemoryRegion* amd_region = (const AMD::MemoryRegion*)region;
[&](const std::shared_ptr<const core::MemoryRegion>& region) {
const AMD::MemoryRegion* amd_region = (const AMD::MemoryRegion*)region.get();
return amd_region->IsLocalMemory() && (!amd_region->fine_grain());
});
return agent_local_region == amd_agent->regions().end() ? nullptr : *agent_local_region;
return agent_local_region == amd_agent->regions().end() ? nullptr : agent_local_region->get();
}
const core::MemoryRegion* RegionMemory::System(bool is_code) {
if (is_code)
return core::Runtime::runtime_singleton_->system_regions_coarse()[0];
return core::Runtime::runtime_singleton_->system_regions_coarse()[0].get();
else
return core::Runtime::runtime_singleton_->system_regions_fine()[0];
return core::Runtime::runtime_singleton_->system_regions_fine()[0].get();
}
bool RegionMemory::Allocate(size_t size, size_t align, bool zero) {
projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_memory_region.cpp
+7 -6
Ver fichero
@@ -48,6 +48,8 @@
#include "core/inc/amd_memory_region.h"
#include <algorithm>
#include <mutex>
#include <shared_mutex>
#include "core/inc/runtime.h"
#include "core/inc/amd_cpu_agent.h"
@@ -132,7 +134,7 @@ MemoryRegion::MemoryRegion(bool fine_grain, bool kernarg, bool full_profile,
MemoryRegion::~MemoryRegion() {}
hsa_status_t MemoryRegion::Allocate(size_t& size, AllocateFlags alloc_flags, void** address, int agent_node_id) const {
ScopedAcquire<KernelMutex> lock(&owner()->agent_memory_lock_);
std::lock_guard<std::mutex> lock(owner()->agent_memory_lock_);
return AllocateImpl(size, alloc_flags, address, agent_node_id);
}
@@ -160,7 +162,7 @@ hsa_status_t MemoryRegion::AllocateImpl(size_t& size, AllocateFlags alloc_flags,
}
hsa_status_t MemoryRegion::Free(void* address, size_t size) const {
ScopedAcquire<KernelMutex> lock(&owner()->agent_memory_lock_);
std::lock_guard<std::mutex> lock(owner()->agent_memory_lock_);
return FreeImpl(address, size);
}
@@ -172,7 +174,7 @@ hsa_status_t MemoryRegion::FreeImpl(void* address, size_t size) const {
// TODO: Look into a better name and/or making this process transparent to exporting.
hsa_status_t MemoryRegion::IPCFragmentExport(void* address) const {
ScopedAcquire<KernelMutex> lock(&owner()->agent_memory_lock_);
std::lock_guard<std::mutex> lock(owner()->agent_memory_lock_);
if (!fragment_allocator_.discardBlock(address)) return HSA_STATUS_ERROR_INVALID_ALLOCATION;
return HSA_STATUS_SUCCESS;
}
@@ -448,7 +450,7 @@ hsa_status_t MemoryRegion::AllowAccess(uint32_t num_agents,
std::vector<uint64_t> union_agents;
info.size = sizeof(info);
ScopedAcquire<KernelMutex> lock(&access_lock_);
std::lock_guard<std::mutex> lock(access_lock_);
if (core::Runtime::runtime_singleton_->PtrInfo(const_cast<void*>(ptr), &info, malloc,
&agent_count, &accessible,
@@ -512,8 +514,7 @@ hsa_status_t MemoryRegion::AllowAccess(uint32_t num_agents,
{ // Sequence with pointer info since queries to other fragments of the block may be adjusted by
// this call.
ScopedAcquire<KernelSharedMutex::Shared> lock(
core::Runtime::runtime_singleton_->memory_lock_.shared());
std::shared_lock<std::shared_mutex> lock(core::Runtime::runtime_singleton_->memory_lock_);
uint64_t alternate_va = 0;
if (owner()->driver().MakeMemoryResident(ptr, size, &alternate_va, &map_flag,
whitelist_nodes.size(),
projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa.cpp
+5 -5
Ver fichero
@@ -1804,7 +1804,7 @@ hsa_status_t hsa_code_object_serialize(
IS_BAD_PTR(serialized_code_object);
IS_BAD_PTR(serialized_code_object_size);
amd::hsa::code::AmdHsaCode *code = GetCodeManager()->FromHandle(code_object);
amd::hsa::code::AmdHsaCode *code = GetCodeManager()->FromHandle(code_object).get();
if (!code) {
return HSA_STATUS_ERROR_INVALID_CODE_OBJECT;
}
@@ -1982,7 +1982,7 @@ hsa_status_t hsa_code_object_get_info(
IS_OPEN();
IS_BAD_PTR(value);
amd::hsa::code::AmdHsaCode *code = GetCodeManager()->FromHandle(code_object);
amd::hsa::code::AmdHsaCode *code = GetCodeManager()->FromHandle(code_object).get();
if (!code) {
return HSA_STATUS_ERROR_INVALID_CODE_OBJECT;
}
@@ -2039,7 +2039,7 @@ hsa_status_t hsa_code_object_get_symbol(
IS_BAD_PTR(symbol_name);
IS_BAD_PTR(symbol);
amd::hsa::code::AmdHsaCode *code = GetCodeManager()->FromHandle(code_object);
amd::hsa::code::AmdHsaCode *code = GetCodeManager()->FromHandle(code_object).get();
if (!code) {
return HSA_STATUS_ERROR_INVALID_CODE_OBJECT;
}
@@ -2059,7 +2059,7 @@ hsa_status_t hsa_code_object_get_symbol_from_name(
IS_BAD_PTR(symbol_name);
IS_BAD_PTR(symbol);
amd::hsa::code::AmdHsaCode *code = GetCodeManager()->FromHandle(code_object);
amd::hsa::code::AmdHsaCode *code = GetCodeManager()->FromHandle(code_object).get();
if (!code) {
return HSA_STATUS_ERROR_INVALID_CODE_OBJECT;
}
@@ -2097,7 +2097,7 @@ hsa_status_t hsa_code_object_iterate_symbols(
IS_OPEN();
IS_BAD_PTR(callback);
amd::hsa::code::AmdHsaCode *code = GetCodeManager()->FromHandle(code_object);
amd::hsa::code::AmdHsaCode *code = GetCodeManager()->FromHandle(code_object).get();
if (!code) {
return HSA_STATUS_ERROR_INVALID_CODE_OBJECT;
}
projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp
+2 -2
Ver fichero
@@ -759,7 +759,7 @@ hsa_status_t hsa_amd_memory_lock(void* host_ptr, size_t size,
}
const AMD::MemoryRegion* system_region = static_cast<const AMD::MemoryRegion*>(
core::Runtime::runtime_singleton_->system_regions_coarse()[0]);
core::Runtime::runtime_singleton_->system_regions_coarse()[0].get());
return system_region->Lock(num_agent, agents, host_ptr, size, 0, agent_ptr);
CATCH;
@@ -799,7 +799,7 @@ hsa_status_t hsa_amd_memory_unlock(void* host_ptr) {
const AMD::MemoryRegion* system_region =
reinterpret_cast<const AMD::MemoryRegion*>(
core::Runtime::runtime_singleton_->system_regions_fine()[0]);
core::Runtime::runtime_singleton_->system_regions_fine()[0].get());
return system_region->Unlock(host_ptr);
CATCH;
projects/rocr-runtime/runtime/hsa-runtime/core/runtime/intercept_queue.cpp
+1 -1
Ver fichero
@@ -340,7 +340,7 @@ void InterceptQueue::StoreRelaxed(hsa_signal_value_t value) {
return;
}
ScopedAcquire<KernelMutex> lock(&lock_);
std::lock_guard<std::mutex> lock(lock_);
// Submit overflow packets.
if (!overflow_.empty()) {
projects/rocr-runtime/runtime/hsa-runtime/core/runtime/interrupt_signal.cpp
+2 -2
Ver fichero
@@ -48,7 +48,7 @@ namespace rocr {
namespace core {
HsaEvent* InterruptSignal::EventPool::alloc() {
ScopedAcquire<HybridMutex> lock(&lock_);
std::lock_guard<HybridMutex> lock(lock_);
if (events_.empty()) {
if (!allEventsAllocated) {
HsaEvent* evt = InterruptSignal::CreateEvent(HSA_EVENTTYPE_SIGNAL, false);
@@ -64,7 +64,7 @@ HsaEvent* InterruptSignal::EventPool::alloc() {
void InterruptSignal::EventPool::free(HsaEvent* evt) {
if (evt == nullptr) return;
ScopedAcquire<HybridMutex> lock(&lock_);
std::lock_guard<HybridMutex> lock(lock_);
events_.push_back(unique_event_ptr(evt));
}
projects/rocr-runtime/runtime/hsa-runtime/core/runtime/ipc_signal.cpp
+2 -2
Ver fichero
@@ -50,7 +50,7 @@
namespace rocr {
namespace core {
KernelMutex IPCSignal::lock_;
std::mutex IPCSignal::lock_;
SharedMemory::SharedMemory(const hsa_amd_ipc_memory_t* handle, size_t len) {
hsa_status_t err = Runtime::runtime_singleton_->IPCAttach(handle, len, 0, NULL, &ptr_);
@@ -85,7 +85,7 @@ Signal* IPCSignal::Attach(const hsa_amd_ipc_signal_t* ipc_signal_handle) {
hsa_signal_t handle = SharedSignal::Convert(shared.signal());
ScopedAcquire<KernelMutex> lock(&lock_);
std::lock_guard<std::mutex> lock(lock_);
Signal* ret = core::Signal::DuplicateHandle(handle);
if (ret == nullptr) ret = new IPCSignal(std::move(shared));
return ret;
projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp
+61 -58
Ver fichero
@@ -48,6 +48,7 @@
#include <string>
#include <vector>
#include <list>
#include <shared_mutex>
#if defined(__linux__)
#include <link.h>
#include <dlfcn.h>
@@ -119,7 +120,7 @@ bool g_use_mwaitx;
Runtime* Runtime::runtime_singleton_ = NULL;
hsa_status_t Runtime::Acquire() {
ScopedAcquire<KernelMutex> boot(&bootstrap_lock());
std::lock_guard<std::mutex> boot(bootstrap_lock());
if (runtime_singleton_ == NULL) {
memset(log_flags, 0, sizeof(log_flags));
@@ -146,7 +147,7 @@ hsa_status_t Runtime::Acquire() {
}
hsa_status_t Runtime::Release() {
ScopedAcquire<KernelMutex> boot(&bootstrap_lock());
std::lock_guard<std::mutex> boot(bootstrap_lock());
if (runtime_singleton_ == nullptr) return HSA_STATUS_ERROR_NOT_INITIALIZED;
@@ -192,7 +193,7 @@ void Runtime::RegisterAgent(Agent* agent, bool Enabled) {
agents_by_gpuid_[0] = agent;
// Add cpu regions to the system region list.
for (const core::MemoryRegion* region : agent->regions()) {
for (auto region : agent->regions()) {
if (region->fine_grain()) {
system_regions_fine_.push_back(region);
} else {
@@ -216,7 +217,7 @@ void Runtime::RegisterAgent(Agent* agent, bool Enabled) {
assert(alignment <= 4096);
void* ptr = NULL;
return (HSA_STATUS_SUCCESS ==
core::Runtime::runtime_singleton_->AllocateMemory(pool, size, alloc_flags,
core::Runtime::runtime_singleton_->AllocateMemory(pool.get(), size, alloc_flags,
&ptr, agent_node_id))
? ptr
: NULL;
@@ -336,7 +337,7 @@ hsa_status_t Runtime::AllocateMemory(const MemoryRegion* region, size_t size,
hsa_status_t status = region->Allocate(size, alloc_flags, address, agent_node_id);
// Track the allocation result so that it could be freed properly.
if (status == HSA_STATUS_SUCCESS) {
ScopedAcquire<KernelSharedMutex> lock(&memory_lock_);
std::lock_guard<std::shared_mutex> lock(memory_lock_);
allocation_map_[*address] = AllocationRegion(region, size, size_requested, alloc_flags);
}
@@ -354,7 +355,7 @@ hsa_status_t Runtime::FreeMemory(void* ptr) {
MemoryRegion::AllocateFlags alloc_flags = core::MemoryRegion::AllocateNoFlags;
{
ScopedAcquire<KernelSharedMutex> lock(&memory_lock_);
std::lock_guard<std::shared_mutex> lock(memory_lock_);
std::map<const void*, AllocationRegion>::iterator it = allocation_map_.find(ptr);
@@ -458,7 +459,7 @@ hsa_status_t Runtime::FreeMemory(void* ptr) {
hsa_status_t Runtime::RegisterReleaseNotifier(void* ptr, hsa_amd_deallocation_callback_t callback,
void* user_data) {
ScopedAcquire<KernelSharedMutex> lock(&memory_lock_);
std::lock_guard<std::shared_mutex> lock(memory_lock_);
auto mem = allocation_map_.upper_bound(ptr);
if (mem != allocation_map_.begin()) {
mem--;
@@ -482,7 +483,7 @@ hsa_status_t Runtime::RegisterReleaseNotifier(void* ptr, hsa_amd_deallocation_ca
hsa_status_t Runtime::DeregisterReleaseNotifier(void* ptr,
hsa_amd_deallocation_callback_t callback) {
hsa_status_t ret = HSA_STATUS_ERROR_INVALID_ARGUMENT;
ScopedAcquire<KernelSharedMutex> lock(&memory_lock_);
std::lock_guard<std::shared_mutex> lock(memory_lock_);
auto mem = allocation_map_.upper_bound(ptr);
if (mem != allocation_map_.begin()) {
mem--;
@@ -552,7 +553,7 @@ hsa_status_t Runtime::CopyMemory(void* dst, const void* src, size_t size) {
// GPU-CPU
// Must ensure that system memory is visible to the GPU during the copy.
const AMD::MemoryRegion* system_region =
static_cast<const AMD::MemoryRegion*>(system_regions_fine_[0]);
static_cast<const AMD::MemoryRegion*>(system_regions_fine_[0].get());
void* gpuPtr = nullptr;
const auto& locked_copy = [&](void*& ptr, core::Agent* locking_agent) {
@@ -698,7 +699,7 @@ hsa_status_t Runtime::AllowAccess(uint32_t num_agents,
size_t alloc_size = 0;
{
ScopedAcquire<KernelSharedMutex> lock(&memory_lock_);
std::lock_guard<std::shared_mutex> lock(memory_lock_);
std::map<const void*, AllocationRegion>::const_iterator it = allocation_map_.find(ptr);
@@ -929,7 +930,7 @@ hsa_status_t Runtime::InteropMap(uint32_t num_agents, Agent** agents,
*size = info.SizeInBytes;
*ptr = info.MemoryAddress;
ScopedAcquire<KernelSharedMutex> lock(&memory_lock_);
std::lock_guard<std::shared_mutex> lock(memory_lock_);
allocation_map_[info.MemoryAddress] = AllocationRegion(
nullptr, info.SizeInBytes, info.SizeInBytes, core::MemoryRegion::AllocateNoFlags);
@@ -1055,7 +1056,7 @@ hsa_status_t Runtime::PtrInfo(const void* ptr, hsa_amd_pointer_info_t* info, voi
{ // memory_lock protects access to the NMappedNodes array and fragment user data since these may
// change with calls to memory APIs.
ScopedAcquire<KernelSharedMutex> lock(&memory_lock_);
std::lock_guard<std::shared_mutex> lock(memory_lock_);
if (VMemoryPtrInfo(ptr, &retInfo, alloc, num_agents_accessible, accessible) ==
HSA_STATUS_SUCCESS) {
@@ -1196,7 +1197,7 @@ hsa_status_t Runtime::PtrInfo(const void* ptr, hsa_amd_pointer_info_t* info, voi
hsa_status_t Runtime::SetPtrInfoData(const void* ptr, void* userptr) {
{ // Use allocation map if possible to handle fragments.
ScopedAcquire<KernelSharedMutex> lock(&memory_lock_);
std::lock_guard<std::shared_mutex> lock(memory_lock_);
const auto& it = allocation_map_.find(ptr);
if (it != allocation_map_.end()) {
it->second.user_ptr = userptr;
@@ -1307,7 +1308,7 @@ void Runtime::AsyncIPCSockServerConnLoop(void*) {
size_t len = 0;
// Search for registered export pointer
ScopedAcquire<KernelMutex> lock(&ipc_sock_server_lock_);
std::lock_guard<std::mutex> lock(ipc_sock_server_lock_);
for (auto& conns : ipc_sock_server_conns_) {
if (conn_handle == conns.first) {
ptr = reinterpret_cast<void *>(conn_handle);
@@ -1372,7 +1373,7 @@ hsa_status_t Runtime::IPCCreate(void* ptr, size_t len, hsa_amd_ipc_memory_t* han
if (useFrag) {
handle->handle[6] |= 0x80000000 | fragOffset;
// Prevent realloction of fragment for better performance.
ScopedAcquire<KernelSharedMutex::Shared> lock(memory_lock_.shared());
std::shared_lock<std::shared_mutex> lock(memory_lock_);
err = allocation_map_[ptr].region->IPCFragmentExport(ptr);
assert(err == HSA_STATUS_SUCCESS && "Region inconsistent with address map.");
}
@@ -1439,7 +1440,7 @@ hsa_status_t Runtime::IPCCreate(void* ptr, size_t len, hsa_amd_ipc_memory_t* han
close(dmabuf_fd);
ScopedAcquire<KernelMutex> lock(&ipc_sock_server_lock_);
std::lock_guard<std::mutex> lock(ipc_sock_server_lock_);
#if defined(__linux__)
if (!ipc_sock_server_conns_.size()) { // create new runtime socket server
struct sockaddr_un address;
@@ -1549,7 +1550,7 @@ int Runtime::IPCClientImport(uint32_t conn_handle, uint64_t dmabuf_fd_handle,
// Store the buffer object handle in allocation map for later use
if (err == HSAKMT_STATUS_SUCCESS) {
ScopedAcquire<KernelSharedMutex> lock(&memory_lock_);
std::lock_guard<std::shared_mutex> lock(memory_lock_);
allocation_map_[*importAddress] =
AllocationRegion(nullptr, *importSize, *importSize, core::MemoryRegion::AllocateNoFlags);
allocation_map_[*importAddress].ldrm_bo = res.buf_handle;
@@ -1579,7 +1580,7 @@ hsa_status_t Runtime::IPCAttach(const hsa_amd_ipc_memory_t* handle, size_t len,
importAddress = reinterpret_cast<uint8_t*>(importAddress) + fragOffset;
len = Min(len, importSize - fragOffset);
}
ScopedAcquire<KernelSharedMutex> lock(&memory_lock_);
std::lock_guard<std::shared_mutex> lock(memory_lock_);
allocation_map_[importAddress] =
AllocationRegion(nullptr, len, len, core::MemoryRegion::AllocateNoFlags);
allocation_map_[importAddress].ldrm_bo = ldrm_bo;
@@ -1711,7 +1712,7 @@ hsa_status_t Runtime::IPCAttach(const hsa_amd_ipc_memory_t* handle, size_t len,
hsa_status_t Runtime::IPCDetach(void* ptr) {
bool ldrmImportCleaned = false;
{ // Handle imported fragments.
ScopedAcquire<KernelSharedMutex> lock(&memory_lock_);
std::unique_lock<std::shared_mutex> lock(memory_lock_);
const auto& it = allocation_map_.find(ptr);
if (it != allocation_map_.end()) {
if (it->second.region != nullptr) return HSA_STATUS_ERROR_INVALID_ARGUMENT;
@@ -1728,7 +1729,7 @@ hsa_status_t Runtime::IPCDetach(void* ptr) {
assert(!"Unimplemented!");
#endif
allocation_map_.erase(it);
lock.Release(); // Can't hold memory lock when using pointer info.
lock.unlock(); // Can't hold memory lock when using pointer info.
PtrInfoBlockData block = {};
hsa_amd_pointer_info_t info = {};
@@ -1954,7 +1955,7 @@ void Runtime::AsyncEventsPool::clear() {
}
Runtime::AsyncEventItem* Runtime::AsyncEventsPool::alloc() {
ScopedAcquire<HybridMutex> lock(&lock_);
std::lock_guard<HybridMutex> lock(lock_);
if (free_list_.empty()) {
AsyncEventItem* block = reinterpret_cast<AsyncEventItem*>(
allocate_()(block_size_ * sizeof(AsyncEventItem), __alignof(AsyncEventItem), core::MemoryRegion::AllocateNonPaged, 0));
@@ -1985,7 +1986,7 @@ void Runtime::AsyncEventsPool::free(AsyncEventItem* ptr) {
if (ptr == nullptr) return;
ptr->~AsyncEventItem();
ScopedAcquire<HybridMutex> lock(&lock_);
std::lock_guard<HybridMutex> lock(lock_);
ifdebug {
bool valid = false;
@@ -2059,33 +2060,33 @@ void Runtime::BindErrorHandlers() {
// Create memory event with manual reset to avoid racing condition
// with driver in case of multiple concurrent VM faults.
vm_fault_event_ = core::InterruptSignal::CreateEvent(HSA_EVENTTYPE_MEMORY, true);
vm_fault_event_.reset(core::InterruptSignal::CreateEvent(HSA_EVENTTYPE_MEMORY, true));
// Create an interrupt signal object to contain the memory event.
// This signal object will be registered with the async handler global
// thread.
vm_fault_signal_ = new core::InterruptSignal(0, vm_fault_event_);
vm_fault_signal_.reset(new core::InterruptSignal(0, vm_fault_event_.get()));
if (!vm_fault_signal_->IsValid() || vm_fault_signal_->EopEvent() == NULL) {
assert(false && "Failed on creating VM fault signal");
return;
}
SetAsyncSignalHandler(core::Signal::Convert(vm_fault_signal_), HSA_SIGNAL_CONDITION_NE, 0,
VMFaultHandler, reinterpret_cast<void*>(vm_fault_signal_));
SetAsyncSignalHandler(core::Signal::Convert(vm_fault_signal_.get()), HSA_SIGNAL_CONDITION_NE, 0,
VMFaultHandler, reinterpret_cast<void*>(vm_fault_signal_.get()));
// Create HW exception event which is for Non-RAS events
hw_exception_event_ = core::InterruptSignal::CreateEvent(HSA_EVENTTYPE_HW_EXCEPTION, true);
hw_exception_event_.reset(core::InterruptSignal::CreateEvent(HSA_EVENTTYPE_HW_EXCEPTION, true));
hw_exception_signal_ = new core::InterruptSignal(0, hw_exception_event_);
hw_exception_signal_.reset(new core::InterruptSignal(0, hw_exception_event_.get()));
if (!hw_exception_signal_->IsValid() || hw_exception_signal_->EopEvent() == NULL) {
assert(false && "Failed on creating HW Exception signal");
return;
}
SetAsyncSignalHandler(core::Signal::Convert(hw_exception_signal_), HSA_SIGNAL_CONDITION_NE, 0,
HwExceptionHandler, reinterpret_cast<void*>(hw_exception_signal_));
SetAsyncSignalHandler(core::Signal::Convert(hw_exception_signal_.get()), HSA_SIGNAL_CONDITION_NE, 0,
HwExceptionHandler, reinterpret_cast<void*>(hw_exception_signal_.get()));
}
bool Runtime::HwExceptionHandler(hsa_signal_value_t val, void* arg) {
@@ -2262,7 +2263,8 @@ bool Runtime::VMFaultHandler(hsa_signal_value_t val, void* arg) {
}
void Runtime::PrintMemoryMapNear(void* ptr) {
runtime_singleton_->memory_lock_.Acquire();
std::unique_lock<std::shared_mutex> lock(runtime_singleton_->memory_lock_);
auto it = runtime_singleton_->allocation_map_.upper_bound(ptr);
for (int i = 0; i < 2; i++) {
if (it != runtime_singleton_->allocation_map_.begin()) it--;
@@ -2287,8 +2289,9 @@ void Runtime::PrintMemoryMapNear(void* ptr) {
it++;
}
fprintf(stderr, "\n");
it = start;
runtime_singleton_->memory_lock_.Release();
it = start;
lock.unlock();
hsa_amd_pointer_info_t info = {};
PtrInfoBlockData block = {};
uint32_t count = 0;
@@ -2408,7 +2411,7 @@ hsa_status_t Runtime::Load() {
BindErrorHandlers();
loader_ = amd::hsa::loader::Loader::Create(&loader_context_);
loader_.reset(amd::hsa::loader::Loader::Create(&loader_context_));
// Load extensions
LoadExtensions();
@@ -2449,8 +2452,8 @@ void Runtime::Unload() {
UnloadTools();
UnloadExtensions();
amd::hsa::loader::Loader::Destroy(loader_);
loader_ = nullptr;
amd::hsa::loader::Loader::Destroy(loader_.get());
loader_.reset();
for(auto nodeAgent: agents_by_node_) {
for (auto agent: nodeAgent.second)
@@ -2462,17 +2465,17 @@ void Runtime::Unload() {
if (vm_fault_signal_ != nullptr) {
vm_fault_signal_->DestroySignal();
vm_fault_signal_ = nullptr;
vm_fault_signal_.reset();
}
core::InterruptSignal::DestroyEvent(vm_fault_event_);
vm_fault_event_ = nullptr;
vm_fault_event_.reset();
if (hw_exception_signal_ != nullptr) {
hw_exception_signal_->DestroySignal();
hw_exception_signal_ = nullptr;
hw_exception_signal_.reset();
}
core::InterruptSignal::DestroyEvent(hw_exception_event_);
hw_exception_event_ = nullptr;
hw_exception_event_.reset();
SharedSignalPool.clear();
@@ -2890,7 +2893,7 @@ void Runtime::AsyncEvents::Clear() {
hsa_status_t Runtime::SetCustomSystemEventHandler(hsa_amd_system_event_callback_t callback,
void* data) {
ScopedAcquire<KernelMutex> lock(&system_event_lock_);
std::lock_guard<std::mutex> lock(system_event_lock_);
system_event_handlers_.push_back(
std::make_pair(AMD::callback_t<hsa_amd_system_event_callback_t>(callback), data));
return HSA_STATUS_SUCCESS;
@@ -2898,7 +2901,7 @@ hsa_status_t Runtime::SetCustomSystemEventHandler(hsa_amd_system_event_callback_
std::vector<std::pair<AMD::callback_t<hsa_amd_system_event_callback_t>, void*>>
Runtime::GetSystemEventHandlers() {
ScopedAcquire<KernelMutex> lock(&system_event_lock_);
std::lock_guard<std::mutex> lock(system_event_lock_);
return system_event_handlers_;
}
@@ -3269,7 +3272,7 @@ hsa_status_t Runtime::SvmPrefetch(void* ptr, size_t size, hsa_agent_t agent,
}
{
ScopedAcquire<KernelMutex> lock(&prefetch_lock_);
std::lock_guard<std::mutex> lock(prefetch_lock_);
// Remove all fully overlapped and trim partially overlapped ranges.
// Get iteration bounds
auto start = prefetch_map_.upper_bound(base);
@@ -3332,7 +3335,7 @@ hsa_status_t Runtime::SvmPrefetch(void* ptr, size_t size, hsa_agent_t agent,
// Remove the prefetch's ranges from the map.
static auto removePrefetchRanges = [](PrefetchOp* op) {
ScopedAcquire<KernelMutex> lock(&Runtime::runtime_singleton_->prefetch_lock_);
std::lock_guard<std::mutex> lock(Runtime::runtime_singleton_->prefetch_lock_);
auto it = op->prefetch_map_entry;
while (it != Runtime::runtime_singleton_->prefetch_map_.end()) {
auto next = it->second.next;
@@ -3389,7 +3392,7 @@ Agent* Runtime::GetSVMPrefetchAgent(void* ptr, size_t size) {
std::vector<std::pair<uintptr_t, uintptr_t>> holes;
ScopedAcquire<KernelMutex> lock(&Runtime::runtime_singleton_->prefetch_lock_);
std::lock_guard<std::mutex> lock(Runtime::runtime_singleton_->prefetch_lock_);
auto start = prefetch_map_.upper_bound(base);
if (start != prefetch_map_.begin()) start--;
auto stop = prefetch_map_.lower_bound(end);
@@ -3441,7 +3444,7 @@ Agent* Runtime::GetSVMPrefetchAgent(void* ptr, size_t size) {
hsa_status_t Runtime::DmaBufExport(const void* ptr, size_t size, int* dmabuf, uint64_t* offset,
uint64_t flags) {
#ifdef __linux__
ScopedAcquire<KernelSharedMutex::Shared> lock(memory_lock_.shared());
std::shared_lock<std::shared_mutex> lock(memory_lock_);
// Lookup containing allocation.
auto mem = allocation_map_.upper_bound(ptr);
if (mem != allocation_map_.begin()) {
@@ -3507,7 +3510,7 @@ hsa_status_t Runtime::VMemoryAddressReserve(void** va, size_t size, uint64_t add
if (!alignment) alignment = rocr::os::PageSize();
ScopedAcquire<KernelSharedMutex> lock(&memory_lock_);
std::lock_guard<std::shared_mutex> lock(memory_lock_);
if (flags & HSA_AMD_VMEM_ADDRESS_NO_REGISTER) {
size_t requested = size + alignment - rocr::os::PageSize();
@@ -3548,7 +3551,7 @@ hsa_status_t Runtime::VMemoryAddressReserve(void** va, size_t size, uint64_t add
}
hsa_status_t Runtime::VMemoryAddressFree(void* va, size_t size) {
ScopedAcquire<KernelSharedMutex> lock(&memory_lock_);
std::lock_guard<std::shared_mutex> lock(memory_lock_);
std::map<const void*, AddressHandle>::iterator it = reserved_address_map_.find(va);
if (it == reserved_address_map_.end()) {
@@ -3580,7 +3583,7 @@ hsa_status_t Runtime::VMemoryHandleCreate(const MemoryRegion* region, size_t siz
if (!IsMultipleOf(size, memRegion->GetPageSize()))
return HSA_STATUS_ERROR_INVALID_ARGUMENT;
ScopedAcquire<KernelSharedMutex> lock(&memory_lock_);
std::lock_guard<std::shared_mutex> lock(memory_lock_);
ThunkHandle user_mode_driver_handle;
hsa_status_t status =
region->Allocate(size, alloc_flags, &user_mode_driver_handle, 0);
@@ -3597,7 +3600,7 @@ hsa_status_t Runtime::VMemoryHandleCreate(const MemoryRegion* region, size_t siz
}
hsa_status_t Runtime::VMemoryHandleRelease(hsa_amd_vmem_alloc_handle_t memoryOnlyHandle) {
ScopedAcquire<KernelSharedMutex> lock(&memory_lock_);
std::lock_guard<std::shared_mutex> lock(memory_lock_);
auto memoryHandleIt = memory_handle_map_.find(MemoryHandle::Convert(memoryOnlyHandle));
if (memoryHandleIt == memory_handle_map_.end()) {
@@ -3628,7 +3631,7 @@ hsa_status_t Runtime::VMemoryHandleMap(void* va, size_t size, size_t in_offset,
uint64_t offset = 0, ret;
uint64_t drm_cpu_addr = 0;
ScopedAcquire<KernelSharedMutex> lock(&memory_lock_);
std::lock_guard<std::shared_mutex> lock(memory_lock_);
auto addressHandle = VMemoryFindReservedAddressHandle(va);
if (addressHandle == nullptr ||
reinterpret_cast<uint8_t*>(va) + size >
@@ -3703,7 +3706,7 @@ hsa_status_t Runtime::VMemoryHandleMap(void* va, size_t size, size_t in_offset,
}
hsa_status_t Runtime::VMemoryHandleUnmap(void* va, size_t size) {
ScopedAcquire<KernelSharedMutex> lock(&memory_lock_);
std::lock_guard<std::shared_mutex> lock(memory_lock_);
std::list<std::pair<void*, MappedHandle*>> mappedHandles;
// va + size may consist of multiple MappedHandle's.
@@ -3921,7 +3924,7 @@ hsa_status_t Runtime::VMemorySetAccess(void* va, size_t size,
if (targetAgent == NULL || !targetAgent->IsValid()) return HSA_STATUS_ERROR_INVALID_AGENT;
}
ScopedAcquire<KernelSharedMutex> lock(&memory_lock_);
std::lock_guard<std::shared_mutex> lock(memory_lock_);
auto addressHandle = VMemoryFindReservedAddressHandle(va);
if (addressHandle == nullptr ||
@@ -4014,7 +4017,7 @@ hsa_status_t Runtime::VMemoryGetAccess(const void* va, hsa_access_permission_t*
*perms = HSA_ACCESS_PERMISSION_NONE;
bool mappedHandleFound = false;
ScopedAcquire<KernelSharedMutex> lock(&memory_lock_);
std::lock_guard<std::shared_mutex> lock(memory_lock_);
auto mappedHandleIt = mapped_handle_map_.upper_bound(va);
if (mappedHandleIt != mapped_handle_map_.begin()) {
@@ -4076,8 +4079,8 @@ hsa_status_t Runtime::VMemoryImportShareableHandle(int dmabuf_fd,
return;
}
for (const core::MemoryRegion* region : agent->regions()) {
const AMD::MemoryRegion* amd_region = reinterpret_cast<const AMD::MemoryRegion*>(region);
for (const auto& region : agent->regions()) {
const AMD::MemoryRegion* amd_region = reinterpret_cast<const AMD::MemoryRegion*>(region.get());
// TODO: Verify that this works on a system with FINE_GRAINED memory.
// System's with FINE_GRAINED will have both COARSE and FINE grain... need to get the
projects/rocr-runtime/runtime/hsa-runtime/core/runtime/signal.cpp
+7 -7
Ver fichero
@@ -58,7 +58,7 @@
namespace rocr {
namespace core {
KernelMutex Signal::ipcLock_;
std::mutex Signal::ipcLock_;
std::map<decltype(hsa_signal_t::handle), Signal*> Signal::ipcMap_;
void SharedSignalPool_t::clear() {
@@ -76,7 +76,7 @@ void SharedSignalPool_t::clear() {
}
SharedSignal* SharedSignalPool_t::alloc() {
ScopedAcquire<HybridMutex> lock(&lock_);
std::lock_guard<HybridMutex> lock(lock_);
if (free_list_.empty()) {
SharedSignal* block = reinterpret_cast<SharedSignal*>(
allocate_()(block_size_ * sizeof(SharedSignal), __alignof(SharedSignal), core::MemoryRegion::AllocateNonPaged, 0));
@@ -109,7 +109,7 @@ void SharedSignalPool_t::free(SharedSignal* ptr) {
if (ptr == nullptr) return;
ptr->~SharedSignal();
ScopedAcquire<HybridMutex> lock(&lock_);
std::lock_guard<HybridMutex> lock(lock_);
ifdebug {
bool valid = false;
@@ -134,7 +134,7 @@ LocalSignal::LocalSignal(hsa_signal_value_t initial_value, bool exportable)
}
void Signal::registerIpc() {
ScopedAcquire<KernelMutex> lock(&ipcLock_);
std::lock_guard<std::mutex> lock(ipcLock_);
auto handle = Convert(this);
assert(ipcMap_.find(handle.handle) == ipcMap_.end() &&
"Can't register the same IPC signal twice.");
@@ -142,7 +142,7 @@ void Signal::registerIpc() {
}
bool Signal::deregisterIpc() {
ScopedAcquire<KernelMutex> lock(&ipcLock_);
std::lock_guard<std::mutex> lock(ipcLock_);
if (refcount_ != 0) return false;
auto handle = Convert(this);
const auto& it = ipcMap_.find(handle.handle);
@@ -152,14 +152,14 @@ bool Signal::deregisterIpc() {
}
Signal* Signal::lookupIpc(hsa_signal_t signal) {
ScopedAcquire<KernelMutex> lock(&ipcLock_);
std::lock_guard<std::mutex> lock(ipcLock_);
const auto& it = ipcMap_.find(signal.handle);
if (it == ipcMap_.end()) return nullptr;
return it->second;
}
Signal* Signal::duplicateIpc(hsa_signal_t signal) {
ScopedAcquire<KernelMutex> lock(&ipcLock_);
std::lock_guard<std::mutex> lock(ipcLock_);
const auto& it = ipcMap_.find(signal.handle);
if (it == ipcMap_.end()) return nullptr;
it->second->refcount_++;