diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/common/shared.h b/projects/rocr-runtime/runtime/hsa-runtime/core/common/shared.h index 900b414fd9..374d770d41 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/common/shared.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/common/shared.h @@ -68,35 +68,89 @@ class BaseShared { static std::function free_; }; -/// @brief Container for object located in GPU visible host memory. -/// Alignment defaults to __alignof(T) but may be increased. -template class Shared final : private BaseShared { +/// @brief Default Allocator for Shared. Ensures allocations are whole pages. +template class PageAllocator : private BaseShared { public: - Shared() { - assert(allocate_ != nullptr && free_ != nullptr && - "Shared object allocator is not set"); - static_assert((__alignof(T) <= Align) || (Align == 0), - "Align is less than alignof(T)"); + __forceinline static T* alloc() { + T* ret = reinterpret_cast(allocate_(AlignUp(sizeof(T), 4096), 4096, 0)); + if (ret == nullptr) throw std::bad_alloc(); - shared_object_ = - reinterpret_cast(allocate_(sizeof(T), Max(__alignof(T), Align), 0)); - if (shared_object_ == nullptr) throw std::bad_alloc(); + MAKE_NAMED_SCOPE_GUARD(throwGuard, [&]() { free_(ret); }); - MAKE_NAMED_SCOPE_GUARD(throwGuard, [&]() { free_(shared_object_); }); - - new (shared_object_) T; + new (ret) T; throwGuard.Dismiss(); + return ret; + } + + __forceinline static void free(T* ptr) { + if (ptr != nullptr) { + ptr->~T(); + free_(ptr); + } + } +}; + +/// @brief Container for object located in GPU visible host memory. +/// If a custom allocator is not given then data will be placed in dedicated pages. +template > +class Shared final : private BaseShared { + public: + explicit Shared(Allocator* pool = nullptr) : pool_(pool) { + assert(allocate_ != nullptr && free_ != nullptr && + "Shared object allocator is not set"); + + if (pool_) + shared_object_ = pool_->alloc(); + else + shared_object_ = PageAllocator::alloc(); + } + + ~Shared() { + assert(allocate_ != nullptr && free_ != nullptr && "Shared object allocator is not set"); + + if (pool_) + pool_->free(shared_object_); + else + PageAllocator::free(shared_object_); + } + + Shared(Shared&& rhs) { + this->~Shared(); + shared_object_ = rhs.shared_object_; + rhs.shared_object_ = nullptr; + pool_ = rhs.pool_; + rhs.pool_ = nullptr; + } + Shared& operator=(Shared&& rhs) { + this->~Shared(); + shared_object_ = rhs.shared_object_; + rhs.shared_object_ = nullptr; + pool_ = rhs.pool_; + rhs.pool_ = nullptr; + return *this; + } + + T* shared_object() const { return shared_object_; } + + private: + T* shared_object_; + Allocator* pool_; +}; + +template class Shared> final : private BaseShared { + public: + Shared() { + assert(allocate_ != nullptr && free_ != nullptr && "Shared object allocator is not set"); + + shared_object_ = PageAllocator::alloc(); } ~Shared() { assert(allocate_ != nullptr && free_ != nullptr && "Shared object allocator is not set"); - if (shared_object_ != nullptr) { - shared_object_->~T(); - free_(shared_object_); - } + PageAllocator::free(shared_object_); } Shared(Shared&& rhs) { diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/default_signal.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/default_signal.h index 20f0f5bb21..a7d9b8afdd 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/default_signal.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/default_signal.h @@ -170,7 +170,7 @@ class DefaultSignal : private LocalSignal, public BusyWaitSignal { /// @brief See base class Signal. explicit DefaultSignal(hsa_signal_value_t initial_value, bool enableIPC = false) - : LocalSignal(initial_value), BusyWaitSignal(signal(), enableIPC) {} + : LocalSignal(initial_value, enableIPC), BusyWaitSignal(signal(), enableIPC) {} protected: bool _IsA(rtti_t id) const { diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/queue.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/queue.h index 06a6911f15..ba8200e826 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/queue.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/queue.h @@ -130,7 +130,7 @@ class LocalQueue { SharedQueue* queue() const { return local_queue_.shared_object(); } private: - Shared local_queue_; + Shared local_queue_; }; /// @brief Class Queue which encapsulate user mode queues and diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/runtime.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/runtime.h index 9c189cd51d..477ee36c3e 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/runtime.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/runtime.h @@ -333,6 +333,8 @@ class Runtime { void InternalQueueCreateNotify(const hsa_queue_t* queue, hsa_agent_t agent); + SharedSignalPool_t* GetSharedSignalPool() { return &SharedSignalPool; } + protected: static void AsyncEventsLoop(void*); @@ -505,6 +507,9 @@ class Runtime { // Track environment variables. Flag flag_; + // Pools memory for SharedSignal (Signal ABI blocks) + SharedSignalPool_t SharedSignalPool; + // Frees runtime memory when the runtime library is unloaded if safe to do so. // Failure to release the runtime indicates an incorrect application but is // common (example: calls library routines at process exit). diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/signal.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/signal.h index 0703a7fe0d..5209fb7430 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/signal.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/signal.h @@ -48,12 +48,13 @@ #include #include #include +#include +#include #include "hsakmt.h" #include "core/common/shared.h" -#include "core/inc/runtime.h" #include "core/inc/checked.h" #include "core/inc/exceptions.h" @@ -112,16 +113,36 @@ static_assert(std::is_standard_layout::value, static_assert(std::is_trivially_destructible::value, "SharedSignal must not be modified on delete for IPC use."); +/// @brief Pool class for SharedSignal suitable for use with Shared. +class SharedSignalPool_t : private BaseShared { + public: + SharedSignalPool_t() : block_size_(minblock_) {} + ~SharedSignalPool_t() { clear(); } + + SharedSignal* alloc(); + void free(SharedSignal* ptr); + void clear(); + + private: + static const size_t minblock_ = 4096 / sizeof(SharedSignal); + KernelMutex lock_; + std::vector free_list_; + std::vector> block_list_; + size_t block_size_; +}; + class LocalSignal { public: + // Temporary, for legacy tools lib support. explicit LocalSignal(hsa_signal_value_t initial_value) { local_signal_.shared_object()->amd_signal.value = initial_value; } + LocalSignal(hsa_signal_value_t initial_value, bool exportable); SharedSignal* signal() const { return local_signal_.shared_object(); } private: - Shared local_signal_; + Shared local_signal_; }; /// @brief An abstract base class which helps implement the public hsa_signal_t diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp index d9ef5d6a66..cd6f341ab7 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp @@ -80,7 +80,7 @@ int AqlQueue::rtti_id_ = 0; AqlQueue::AqlQueue(GpuAgent* agent, size_t req_size_pkts, HSAuint32 node_id, ScratchInfo& scratch, core::HsaEventCallback callback, void* err_data, bool is_kv) : Queue(), - LocalSignal(0), + LocalSignal(0, false), DoorbellSignal(signal()), ring_buf_(nullptr), ring_buf_alloc_bytes_(0), diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/intercept_queue.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/intercept_queue.cpp index 4761a8f4f9..017c505f46 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/intercept_queue.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/intercept_queue.cpp @@ -69,7 +69,7 @@ int InterceptQueue::rtti_id_ = 0; InterceptQueue::InterceptQueue(std::unique_ptr queue) : QueueProxy(std::move(queue)), - LocalSignal(0), + LocalSignal(0, false), DoorbellSignal(signal()), next_packet_(0), retry_index_(0), diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/interrupt_signal.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/interrupt_signal.cpp index c22a906044..08ef9dd4f4 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/interrupt_signal.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/interrupt_signal.cpp @@ -69,7 +69,7 @@ int InterruptSignal::rtti_id_ = 0; void InterruptSignal::DestroyEvent(HsaEvent* evt) { hsaKmtDestroyEvent(evt); } InterruptSignal::InterruptSignal(hsa_signal_value_t initial_value, HsaEvent* use_event) - : LocalSignal(initial_value), Signal(signal()) { + : LocalSignal(initial_value, false), Signal(signal()) { if (use_event != NULL) { event_ = use_event; free_event_ = false; diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp index bd9db6a294..772b8310ae 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp @@ -1219,6 +1219,8 @@ void Runtime::Unload() { core::InterruptSignal::DestroyEvent(vm_fault_event_); vm_fault_event_ = nullptr; + SharedSignalPool.clear(); + DestroyAgents(); CloseTools(); diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/signal.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/signal.cpp index f18c038365..f76d00404d 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/signal.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/signal.cpp @@ -47,12 +47,84 @@ #include #include "core/util/timer.h" +#include "core/inc/runtime.h" namespace core { KernelMutex Signal::ipcLock_; std::map Signal::ipcMap_; +void SharedSignalPool_t::clear() { + ifdebug { + size_t capacity = 0; + for (auto& block : block_list_) capacity += block.second; + if (capacity != free_list_.size()) + debug_print("Warning: Resource leak detected by SharedSignalPool, %ld Signals leaked.\n", + capacity - free_list_.size()); + } + + for (auto& block : block_list_) free_(block.first); + block_list_.clear(); + free_list_.clear(); +} + +SharedSignal* SharedSignalPool_t::alloc() { + ScopedAcquire lock(&lock_); + if (free_list_.empty()) { + SharedSignal* block = reinterpret_cast( + allocate_(block_size_ * sizeof(SharedSignal), __alignof(SharedSignal), 0)); + if (block == nullptr) { + block_size_ = minblock_; + block = reinterpret_cast( + allocate_(block_size_ * sizeof(SharedSignal), __alignof(SharedSignal), 0)); + if (block == nullptr) throw std::bad_alloc(); + } + + MAKE_NAMED_SCOPE_GUARD(throwGuard, [&]() { free_(block); }); + block_list_.push_back(std::make_pair(block, block_size_)); + throwGuard.Dismiss(); + + + for (int i = 0; i < block_size_; i++) { + free_list_.push_back(&block[i]); + } + + block_size_ *= 2; + } + + SharedSignal* ret = free_list_.back(); + new (ret) SharedSignal(); + free_list_.pop_back(); + return ret; +} + +void SharedSignalPool_t::free(SharedSignal* ptr) { + if (ptr == nullptr) return; + + ptr->~SharedSignal(); + ScopedAcquire lock(&lock_); + + ifdebug { + bool valid = false; + for (auto& block : block_list_) { + if ((block.first <= ptr) && + (uintptr_t(ptr) < uintptr_t(block.first) + block.second * sizeof(SharedSignal))) { + valid = true; + break; + } + } + assert(valid && "Object does not belong to pool."); + } + + free_list_.push_back(ptr); +} + +LocalSignal::LocalSignal(hsa_signal_value_t initial_value, bool exportable) + : local_signal_(exportable ? nullptr + : core::Runtime::runtime_singleton_->GetSharedSignalPool()) { + local_signal_.shared_object()->amd_signal.value = initial_value; +} + void Signal::registerIpc() { ScopedAcquire lock(&ipcLock_); auto handle = Convert(this); diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h b/projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h index 8d0e1af560..d6375617d9 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h @@ -86,11 +86,12 @@ class Flag { tools_lib_names_ = os::GetEnvVar("HSA_TOOLS_LIB"); var = os::GetEnvVar("HSA_TOOLS_REPORT_LOAD_FAILURE"); -#ifdef NDEBUG - report_tool_load_failures_ = (var == "1") ? true : false; -#else - report_tool_load_failures_ = (var == "0") ? false : true; -#endif + + ifdebug { + report_tool_load_failures_ = (var == "1") ? true : false; + } else { + report_tool_load_failures_ = (var == "0") ? false : true; + } var = os::GetEnvVar("HSA_DISABLE_FRAGMENT_ALLOCATOR"); disable_fragment_alloc_ = (var == "1") ? true : false; diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/util/utils.h b/projects/rocr-runtime/runtime/hsa-runtime/core/util/utils.h index 8a18b06ce5..a512c54f24 100755 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/util/utils.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/util/utils.h @@ -121,6 +121,12 @@ static __forceinline unsigned long long int strtoull(const char* str, } while (false) #endif +#ifdef NDEBUG +#define ifdebug if (false) +#else +#define ifdebug if (true) +#endif + // A macro to disallow the copy and move constructor and operator= functions #define DISALLOW_COPY_AND_ASSIGN(TypeName) \ TypeName(const TypeName&) = delete; \