diff --git a/rocclr/device/rocm/rocvirtual.cpp b/rocclr/device/rocm/rocvirtual.cpp index 78ebece0d5..8e5f3e9dcd 100644 --- a/rocclr/device/rocm/rocvirtual.cpp +++ b/rocclr/device/rocm/rocvirtual.cpp @@ -109,12 +109,14 @@ static unsigned extractAqlBits(unsigned v, unsigned pos, unsigned width) { }; // ================================================================================================ -void Timestamp::checkGpuTime() { +void Timestamp::checkGpuTime(bool event_recycle) { if (HwProfiling()) { uint64_t start = std::numeric_limits::max(); uint64_t end = 0; for (auto it : signals_) { + amd::ScopedLock lock(it->LockSignalOps()); + // Ignore the wait if runtime processes API callback, because the signal value is bigger // than expected and the value reset will occur after API callback is done if (GetCallbackSignal().handle == 0) { @@ -138,6 +140,10 @@ void Timestamp::checkGpuTime() { ClPrint(amd::LOG_INFO, amd::LOG_SIG, "Signal = (0x%lx), start = %ld, " "end = %ld", it->signal_.handle, start, end); } + // The signal is reused and the upper layer can't rely on it. + if (event_recycle) { + const_cast(it->ts_->command_).SetHwEvent(nullptr); + } it->ts_ = nullptr; it->done_ = true; } @@ -325,7 +331,7 @@ VirtualGPU::HwQueueTracker::~HwQueueTracker() { // ================================================================================================ bool VirtualGPU::HwQueueTracker::Create() { - constexpr size_t kSignalListSize = 16; + constexpr size_t kSignalListSize = 32; signal_list_.resize(kSignalListSize); hsa_agent_t agent = gpu_.gpu_device(); @@ -475,11 +481,13 @@ std::vector& VirtualGPU::HwQueueTracker::WaitingSignal(HwQueueEngi // ================================================================================================ bool VirtualGPU::HwQueueTracker::CpuWaitForSignal(ProfilingSignal* signal) { + amd::ScopedLock lock(signal->LockSignalOps()); // Wait for the current signal if (!signal->done_) { // Update timestamp values if requested if (signal->ts_ != nullptr) { - signal->ts_->checkGpuTime(); + static constexpr bool kEventRecycle = true; + signal->ts_->checkGpuTime(kEventRecycle); } else { ClPrint(amd::LOG_DEBUG, amd::LOG_COPY, "[%zx]!\t Host wait on completion_signal=0x%zx", std::this_thread::get_id(), signal->signal_.handle); diff --git a/rocclr/device/rocm/rocvirtual.hpp b/rocclr/device/rocm/rocvirtual.hpp index 4d40023d94..aaa05acf67 100644 --- a/rocclr/device/rocm/rocvirtual.hpp +++ b/rocclr/device/rocm/rocvirtual.hpp @@ -37,15 +37,18 @@ class Memory; class Timestamp; struct ProfilingSignal : public amd::HeapObject { + amd::Monitor lock_; //!< Signal lock for update hsa_signal_t signal_; //!< HSA signal to track profiling information Timestamp* ts_; //!< Timestamp object associated with the signal HwQueueEngine engine_; //!< Engine used with this signal bool done_; //!< True if signal is done ProfilingSignal() - : ts_(nullptr) + : lock_("Signal Ops Lock", true) + , ts_(nullptr) , engine_(HwQueueEngine::Compute) , done_(true) { signal_.handle = 0; } + amd::Monitor& LockSignalOps() { return lock_; } }; // Initial HSA signal value @@ -136,7 +139,7 @@ class Timestamp : public amd::HeapObject { const bool HwProfiling() const { return !signals_.empty(); } //! Finds execution ticks on GPU - void checkGpuTime(); + void checkGpuTime(bool event_recycle = false); // Start a timestamp (get timestamp from OS) void start() { start_ = amd::Os::timeNanos(); }