Andryeyev, German
6df9a49437
* SWDEV-465041 - Add support for user events with DD
User events can be replaced with HSA signals. Add the interface
to allocate HSA signal for user events and update the status on
CL_COMPLETE.
Force pinned path with DD to avoid blocking calls. Pinned memory
can be released only when the command is complete.
Simplify device enqueue path to use generic kernel arg buffer and
signals
* Fix notifyCmdQueue() logic for OCL
* Avoid blocking calls in OCL with DD
* Add event destruciton in a case of the failure.
[ROCm/clr commit: 2305f8ae56]
823 строки
29 KiB
C++
823 строки
29 KiB
C++
/* Copyright (c) 2008 - 2024 Advanced Micro Devices, Inc.
|
|
|
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
of this software and associated documentation files (the "Software"), to deal
|
|
in the Software without restriction, including without limitation the rights
|
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
copies of the Software, and to permit persons to whom the Software is
|
|
furnished to do so, subject to the following conditions:
|
|
|
|
The above copyright notice and this permission notice shall be included in
|
|
all copies or substantial portions of the Software.
|
|
|
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
THE SOFTWARE. */
|
|
|
|
#include "platform/activity.hpp"
|
|
#include "platform/command.hpp"
|
|
#include "platform/commandqueue.hpp"
|
|
#include "device/device.hpp"
|
|
#include "platform/context.hpp"
|
|
#include "platform/kernel.hpp"
|
|
#include "thread/monitor.hpp"
|
|
#include "platform/memory.hpp"
|
|
#include "platform/agent.hpp"
|
|
#include "os/alloc.hpp"
|
|
|
|
#include <atomic>
|
|
#include <cstring>
|
|
#include <algorithm>
|
|
|
|
namespace amd {
|
|
|
|
// ================================================================================================
|
|
Event::Event(HostQueue& queue, bool profilingEnabled)
|
|
: callbacks_(NULL),
|
|
status_(CL_INT_MAX),
|
|
hw_event_(nullptr),
|
|
notify_event_(nullptr),
|
|
device_(&queue.device()),
|
|
profilingInfo_(profilingEnabled) {
|
|
event_entry_scope_.store(Device::kCacheStateInvalid, std::memory_order_relaxed);
|
|
notified_.clear();
|
|
}
|
|
|
|
// ================================================================================================
|
|
Event::Event()
|
|
: callbacks_(NULL),
|
|
status_(CL_SUBMITTED),
|
|
hw_event_(nullptr),
|
|
notify_event_(nullptr),
|
|
device_(nullptr) {
|
|
event_entry_scope_.store(Device::kCacheStateInvalid, std::memory_order_relaxed);
|
|
notified_.clear();
|
|
}
|
|
|
|
// ================================================================================================
|
|
Event::~Event() {
|
|
CallBackEntry* callback = callbacks_;
|
|
while (callback != NULL) {
|
|
CallBackEntry* next = callback->next_;
|
|
delete callback;
|
|
callback = next;
|
|
}
|
|
// Release the notify event
|
|
if (notify_event_ != nullptr) {
|
|
notify_event_->release();
|
|
}
|
|
// Destroy global HW event if available
|
|
if ((hw_event_ != nullptr) && (device_ != nullptr)) {
|
|
device_->ReleaseGlobalSignal(hw_event_);
|
|
}
|
|
}
|
|
|
|
// ================================================================================================
|
|
uint64_t Event::recordProfilingInfo(int32_t status, uint64_t timeStamp) {
|
|
if (timeStamp == 0) {
|
|
timeStamp = Os::timeNanos();
|
|
}
|
|
switch (status) {
|
|
case CL_QUEUED:
|
|
profilingInfo_.queued_ = timeStamp;
|
|
break;
|
|
case CL_SUBMITTED:
|
|
profilingInfo_.submitted_ = timeStamp;
|
|
break;
|
|
case CL_RUNNING:
|
|
profilingInfo_.start_ = timeStamp;
|
|
break;
|
|
default:
|
|
profilingInfo_.end_ = timeStamp;
|
|
break;
|
|
}
|
|
return timeStamp;
|
|
}
|
|
|
|
// Global epoch time since the first processed command
|
|
uint64_t epoch = 0;
|
|
// ================================================================================================
|
|
bool Event::setStatus(int32_t status, uint64_t timeStamp) {
|
|
assert(status <= CL_QUEUED && "invalid status");
|
|
|
|
int32_t currentStatus = this->status();
|
|
if (currentStatus <= CL_COMPLETE || currentStatus <= status) {
|
|
// We can only move forward in the execution status.
|
|
return false;
|
|
}
|
|
|
|
if (profilingInfo().enabled_) {
|
|
timeStamp = recordProfilingInfo(status, timeStamp);
|
|
if (epoch == 0) {
|
|
epoch = profilingInfo().queued_;
|
|
}
|
|
}
|
|
|
|
if (amd::IS_HIP) {
|
|
// HIP API doesn't have any event, associated with a callback. Hence the SW status of
|
|
// the event is irrelevant, during the actual callback. At the same time HIP API requires
|
|
// to finish the callback before HIP stream can continue. Hence runtime has to process
|
|
// the callback first and then update the status.
|
|
if (callbacks_ != (CallBackEntry*)0) {
|
|
processCallbacks(status);
|
|
}
|
|
if (!status_.compare_exchange_strong(currentStatus, status, std::memory_order_relaxed)) {
|
|
// Somebody else beat us to it, let them deal with the release/signal.
|
|
return false;
|
|
}
|
|
} else {
|
|
if (!status_.compare_exchange_strong(currentStatus, status, std::memory_order_relaxed)) {
|
|
// Somebody else beat us to it, let them deal with the release/signal.
|
|
return false;
|
|
}
|
|
if (callbacks_ != (CallBackEntry*)0) {
|
|
processCallbacks(status);
|
|
}
|
|
}
|
|
|
|
if (Agent::shouldPostEventEvents() && command().type() != 0) {
|
|
Agent::postEventStatusChanged(as_cl(this), status, timeStamp + Os::offsetToEpochNanos());
|
|
}
|
|
|
|
if (status <= CL_COMPLETE) {
|
|
// Before we notify the waiters that this event reached the CL_COMPLETE
|
|
// status, we release all the resources associated with this instance.
|
|
if (!IS_HIP) {
|
|
releaseResources();
|
|
}
|
|
|
|
if (profilingInfo().enabled_ && amd::activity_prof::IsEnabled(OP_ID_DISPATCH)) {
|
|
amd::activity_prof::ReportActivity(command());
|
|
}
|
|
|
|
// Broadcast all the waiters.
|
|
if (referenceCount() > 1) {
|
|
signal();
|
|
}
|
|
|
|
if (profilingInfo().enabled_) {
|
|
ClPrint(LOG_DEBUG, LOG_CMD, "Command %p complete (Wall: %ld, CPU: %ld, GPU: %ld us)",
|
|
&command(),
|
|
((profilingInfo().end_ - epoch) / 1000),
|
|
((profilingInfo().submitted_ - profilingInfo().queued_) / 1000),
|
|
((profilingInfo().end_ - profilingInfo().start_) / 1000));
|
|
} else {
|
|
ClPrint(LOG_DEBUG, LOG_CMD, "Command %p complete", &command());
|
|
}
|
|
release();
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
// ================================================================================================
|
|
bool Event::resetStatus(int32_t status) {
|
|
int32_t currentStatus = this->status();
|
|
if (currentStatus != CL_COMPLETE) {
|
|
ClPrint(LOG_ERROR, LOG_CMD, "command is reset before complete current status :%d",
|
|
currentStatus);
|
|
}
|
|
if (!status_.compare_exchange_strong(currentStatus, status, std::memory_order_relaxed)) {
|
|
ClPrint(LOG_ERROR, LOG_CMD, "Failed to reset command status");
|
|
return false;
|
|
}
|
|
notified_.clear();
|
|
return true;
|
|
}
|
|
|
|
// ================================================================================================
|
|
bool Event::setCallback(int32_t status, Event::CallBackFunction callback, void* data,
|
|
bool blocking) {
|
|
assert(status >= CL_COMPLETE && status <= CL_QUEUED && "invalid status");
|
|
|
|
CallBackEntry* entry = new CallBackEntry(status, callback, data, blocking);
|
|
if (entry == NULL) {
|
|
return false;
|
|
}
|
|
|
|
entry->next_ = callbacks_;
|
|
while (!callbacks_.compare_exchange_weak(entry->next_, entry))
|
|
; // Someone else is also updating the head of the linked list! reload.
|
|
|
|
// Check if the event has already reached 'status'
|
|
if (this->status() <= status && entry->callback_ != CallBackFunction(0)) {
|
|
if (entry->callback_.exchange(NULL) != NULL) {
|
|
callback(as_cl(this), status, entry->data_);
|
|
}
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
// ================================================================================================
|
|
void Event::processCallbacks(int32_t status) const {
|
|
cl_event event = const_cast<cl_event>(as_cl(this));
|
|
const int32_t mask = (status > CL_COMPLETE) ? status : CL_COMPLETE;
|
|
|
|
// For_each callback:
|
|
CallBackEntry* entry;
|
|
for (entry = callbacks_; entry != NULL; entry = entry->next_) {
|
|
// If the entry's status matches the mask,
|
|
if (entry->status_ == mask && entry->callback_ != CallBackFunction(0)) {
|
|
// invoke the callback function.
|
|
CallBackFunction callback = entry->callback_.exchange(NULL);
|
|
if (callback != NULL) {
|
|
callback(event, status, entry->data_);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
static constexpr bool kCpuWait = true;
|
|
// ================================================================================================
|
|
bool Event::awaitCompletion() {
|
|
if (status() > CL_COMPLETE) {
|
|
// Notifies the current command queue about waiting
|
|
if (!notifyCmdQueue(kCpuWait)) {
|
|
return false;
|
|
}
|
|
|
|
ClPrint(LOG_DEBUG, LOG_WAIT, "Waiting for event %p to complete, current status %d",
|
|
this, status());
|
|
auto* queue = command().queue();
|
|
if ((queue != nullptr) && queue->vdev()->ActiveWait()) {
|
|
while (status() > CL_COMPLETE) {
|
|
amd::Os::yield();
|
|
}
|
|
} else {
|
|
ScopedLock lock(lock_);
|
|
|
|
// Wait until the status becomes CL_COMPLETE or negative.
|
|
while (status() > CL_COMPLETE) {
|
|
lock_.wait();
|
|
}
|
|
}
|
|
ClPrint(LOG_DEBUG, LOG_WAIT, "Event %p wait completed", this);
|
|
}
|
|
|
|
return status() == CL_COMPLETE;
|
|
}
|
|
|
|
// ================================================================================================
|
|
bool Event::notifyCmdQueue(bool cpu_wait) {
|
|
HostQueue* queue = command().queue();
|
|
if (AMD_DIRECT_DISPATCH) {
|
|
ScopedLock l(notify_lock_);
|
|
if ((status() > CL_COMPLETE) && (nullptr != queue) &&
|
|
// If HW event was assigned, then notification can be ignored, since a barrier was issued
|
|
// @note: Force the marker always in OCL for now, since OCL events require precise
|
|
// sequence of the status update
|
|
((HwEvent() == nullptr) || !amd::IS_HIP) &&
|
|
!notified_.test_and_set()) {
|
|
// Make sure the queue is draining the enqueued commands.
|
|
amd::Command* command = new amd::Marker(*queue, false, nullWaitList, this, cpu_wait);
|
|
if (command == NULL) {
|
|
notified_.clear();
|
|
return false;
|
|
}
|
|
command->enqueue();
|
|
// Save notification, associated with the current event
|
|
notify_event_ = command;
|
|
}
|
|
} else {
|
|
if ((status() > CL_COMPLETE) && (nullptr != queue) && !notified_.test_and_set()) {
|
|
// Make sure the queue is draining the enqueued commands.
|
|
amd::Command* command = new amd::Marker(*queue, false, nullWaitList, this);
|
|
if (command == NULL) {
|
|
notified_.clear();
|
|
return false;
|
|
}
|
|
command->enqueue();
|
|
command->release();
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
const Event::EventWaitList Event::nullWaitList(0);
|
|
|
|
// ================================================================================================
|
|
Command::Command(HostQueue& queue, cl_command_type type, const EventWaitList& eventWaitList,
|
|
uint32_t commandWaitBits, const Event* waitingEvent)
|
|
: Event(queue,
|
|
amd::activity_prof::IsEnabled(amd::activity_prof::OperationId(type)) ||
|
|
queue.properties().test(CL_QUEUE_PROFILING_ENABLE) ||
|
|
Agent::shouldPostEventEvents()),
|
|
queue_(&queue),
|
|
next_(nullptr),
|
|
type_(type),
|
|
waitingEvent_(waitingEvent),
|
|
eventWaitList_(eventWaitList),
|
|
commandWaitBits_(commandWaitBits) {
|
|
// Retain the commands from the event wait list.
|
|
for (const auto &event: eventWaitList) {
|
|
event->retain();
|
|
}
|
|
}
|
|
|
|
SysmemPool<ComputeCommand>* Command::command_pool_ = new SysmemPool<ComputeCommand>;
|
|
// ================================================================================================
|
|
void Command::operator delete(void* ptr) {
|
|
if (DEBUG_CLR_SYSMEM_POOL) {
|
|
command_pool_->Free(ptr);
|
|
} else {
|
|
::operator delete (ptr);
|
|
}
|
|
}
|
|
|
|
// ================================================================================================
|
|
void* Command::operator new(size_t size) {
|
|
if (DEBUG_CLR_SYSMEM_POOL) {
|
|
return command_pool_->Alloc(size);
|
|
} else {
|
|
return ::operator new (size);
|
|
}
|
|
}
|
|
|
|
// ================================================================================================
|
|
void Command::releaseResources() {
|
|
const Command::EventWaitList& events = eventWaitList();
|
|
|
|
// Release the commands from the event wait list.
|
|
for (const auto &event: events) {
|
|
event->release();
|
|
}
|
|
}
|
|
|
|
// ================================================================================================
|
|
void Command::enqueue() {
|
|
assert(queue_ != NULL && "Cannot be enqueued");
|
|
|
|
if (Agent::shouldPostEventEvents() && type_ != 0) {
|
|
Agent::postEventCreate(as_cl(static_cast<Event*>(this)), type_);
|
|
}
|
|
|
|
ClPrint(LOG_DEBUG, LOG_CMD, "Command (%s) enqueued: %p to queue: %p",
|
|
amd::activity_prof::getOclCommandKindString(this->type()), this, queue_);
|
|
|
|
// Direct dispatch logic below will submit the command immediately, but the command status
|
|
// update will occur later after flush() with a wait
|
|
if (AMD_DIRECT_DISPATCH) {
|
|
setStatus(CL_QUEUED);
|
|
|
|
// Notify all commands about the waiter. Barrier will be sent in order to obtain
|
|
// HSA signal for a wait on the current queue
|
|
for (const auto &event: eventWaitList()) {
|
|
if (!amd::IS_HIP && event->command().type() == CL_COMMAND_USER) {
|
|
if (event->status() >= CL_COMPLETE) {
|
|
reinterpret_cast<amd::UserEvent*>(event)->AddDependent(this);
|
|
} else {
|
|
setStatus(CL_EXEC_STATUS_ERROR_FOR_EVENTS_IN_WAIT_LIST);
|
|
return;
|
|
}
|
|
} else {
|
|
event->notifyCmdQueue(!kCpuWait);
|
|
}
|
|
}
|
|
|
|
// The batch update must be lock protected to avoid a race condition
|
|
// when multiple threads submit/flush/update the batch at the same time
|
|
ScopedLock sl(queue_->vdev()->execution());
|
|
queue_->FormSubmissionBatch(this);
|
|
|
|
// Enqueue flushes, except profiling markers to avoid frequent expensive callbacks
|
|
if (((type() == 0) && profilingInfo().batch_flush_) ||
|
|
(type() == CL_COMMAND_MARKER) || (type() == CL_COMMAND_TASK)) {
|
|
// The current HSA signal tracking logic requires profiling enabled for the markers
|
|
EnableProfiling();
|
|
// Update batch head for the current marker. Hence the status of all commands can be
|
|
// updated upon the marker completion
|
|
SetBatchHead(queue_->GetSubmissionBatch());
|
|
|
|
submit(*queue_->vdev());
|
|
|
|
// The batch will be tracked with the marker now
|
|
queue_->ResetSubmissionBatch();
|
|
} else {
|
|
submit(*queue_->vdev());
|
|
queue_->FlushSubmissionBatch(this);
|
|
}
|
|
} else {
|
|
queue_->append(*this);
|
|
queue_->flush();
|
|
}
|
|
|
|
if ((queue_->device().settings().waitCommand_ && (type_ != 0)) ||
|
|
((commandWaitBits_ & 0x2) != 0)) {
|
|
queue_->finish();
|
|
}
|
|
|
|
// set this queue status is active
|
|
queue_->SetQueueStatus();
|
|
}
|
|
|
|
// ================================================================================================
|
|
const Context& Command::context() const { return queue_->context(); }
|
|
|
|
NDRangeKernelCommand::NDRangeKernelCommand(HostQueue& queue, const EventWaitList& eventWaitList,
|
|
Kernel& kernel, const NDRangeContainer& sizes,
|
|
uint32_t sharedMemBytes, uint32_t extraParam,
|
|
uint32_t gridId, uint32_t numGrids,
|
|
uint64_t prevGridSum, uint64_t allGridSum,
|
|
uint32_t firstDevice, bool forceProfiling) :
|
|
Command(queue, CL_COMMAND_NDRANGE_KERNEL, eventWaitList, AMD_SERIALIZE_KERNEL |
|
|
(HIP_LAUNCH_BLOCKING << 1)),
|
|
kernel_(kernel),
|
|
sizes_(sizes),
|
|
sharedMemBytes_(sharedMemBytes),
|
|
extraParam_(extraParam),
|
|
gridId_(gridId),
|
|
numGrids_(numGrids),
|
|
prevGridSum_(prevGridSum),
|
|
allGridSum_(allGridSum),
|
|
firstDevice_(firstDevice) {
|
|
auto& device = queue.device();
|
|
auto devKernel = const_cast<device::Kernel*>(kernel.getDeviceKernel(device));
|
|
if (cooperativeGroups()) {
|
|
setNumWorkgroups();
|
|
}
|
|
|
|
// This optimization will set marker_ts_ but may not submit a batch.
|
|
if (forceProfiling) {
|
|
profilingInfo_.enabled_ = true;
|
|
profilingInfo_.clear();
|
|
profilingInfo_.correlation_id_ = activity_prof::correlation_id;
|
|
profilingInfo_.marker_ts_ = true;
|
|
}
|
|
kernel_.retain();
|
|
}
|
|
|
|
void NDRangeKernelCommand::releaseResources() {
|
|
kernel_.parameters().release(parameters_);
|
|
DEBUG_ONLY(parameters_ = NULL);
|
|
kernel_.release();
|
|
Command::releaseResources();
|
|
}
|
|
|
|
NativeFnCommand::NativeFnCommand(HostQueue& queue, const EventWaitList& eventWaitList,
|
|
void(CL_CALLBACK* nativeFn)(void*), const void* args,
|
|
size_t argsSize, size_t numMemObjs, const cl_mem* memObjs,
|
|
const void** memLocs)
|
|
: Command(queue, CL_COMMAND_NATIVE_KERNEL, eventWaitList),
|
|
nativeFn_(nativeFn),
|
|
argsSize_(argsSize) {
|
|
args_ = new char[argsSize_];
|
|
if (args_ == NULL) {
|
|
return;
|
|
}
|
|
::memcpy(args_, args, argsSize_);
|
|
|
|
memObjects_.resize(numMemObjs);
|
|
memOffsets_.resize(numMemObjs);
|
|
for (size_t i = 0; i < numMemObjs; ++i) {
|
|
Memory* obj = as_amd(memObjs[i]);
|
|
|
|
obj->retain();
|
|
memObjects_[i] = obj;
|
|
memOffsets_[i] = (const_address)memLocs[i] - (const_address)args;
|
|
}
|
|
}
|
|
|
|
int32_t NativeFnCommand::invoke() {
|
|
size_t numMemObjs = memObjects_.size();
|
|
for (size_t i = 0; i < numMemObjs; ++i) {
|
|
void* hostMemPtr = memObjects_[i]->getHostMem();
|
|
if (hostMemPtr == NULL) {
|
|
return CL_MEM_OBJECT_ALLOCATION_FAILURE;
|
|
}
|
|
*reinterpret_cast<void**>(&args_[memOffsets_[i]]) = hostMemPtr;
|
|
}
|
|
nativeFn_(args_);
|
|
return CL_SUCCESS;
|
|
}
|
|
|
|
bool OneMemoryArgCommand::validatePeerMemory() {
|
|
amd::Device* queue_device = &queue()->device();
|
|
// Rocr backend maps memory from different devices by default and runtime doesn't need to track
|
|
// extra memory objects.
|
|
if (queue_device->settings().rocr_backend_) {
|
|
const std::vector<Device*>& srcDevices = memory_->getContext().devices();
|
|
if (!memory_->isArena() &&
|
|
srcDevices.size() == 1 && queue_device != srcDevices[0]) {
|
|
// current device and source device are not same hence
|
|
// explicit allow access is needed for P2P access
|
|
device::Memory* mem = memory_->getDeviceMemory(*srcDevices[0]);
|
|
if (!mem->getAllowedPeerAccess()) {
|
|
void* dst = reinterpret_cast<void*>(mem->virtualAddress());
|
|
bool status = srcDevices[0]->deviceAllowAccess(dst);
|
|
mem->setAllowedPeerAccess(true);
|
|
return status;
|
|
}
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool OneMemoryArgCommand::validateMemory() {
|
|
// Runtime disables deferred memory allocation for single device.
|
|
// Hence ignore memory validations
|
|
if (queue()->context().devices().size() == 1) {
|
|
return true;
|
|
}
|
|
device::Memory* mem = memory_->getDeviceMemory(queue()->device());
|
|
if (NULL == mem) {
|
|
LogPrintfError("Can't allocate memory size - 0x%08X bytes!", memory_->getSize());
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool TwoMemoryArgsCommand::validatePeerMemory(){
|
|
bool accessAllowed = true;
|
|
amd::Device* queue_device = &queue()->device();
|
|
// Explicite Allow access is needed when first time memory is accessed from other device.
|
|
// Rules : Remote device has to provide access to current device
|
|
// --------------------------------------------------------------------
|
|
// Crr_Dev = src | Allow access will be called for dst memory |
|
|
// --------------------------------------------------------------------
|
|
// Crr_Dev = dst | Allow access will be called for src memory |
|
|
// --------------------------------------------------------------------
|
|
// Crr_dev = other| Allow access will be called for dst and src memory|
|
|
// --------------------------------------------------------------------
|
|
if (queue_device->settings().rocr_backend_) {
|
|
const std::vector<Device*>& srcDevices = memory1_->getContext().devices();
|
|
const std::vector<Device*>& dstDevices = memory2_->getContext().devices();
|
|
// explicit allow access is needed for P2P access
|
|
device::Memory* mem1 = memory1_->getDeviceMemory(*srcDevices[0]);
|
|
if (!memory1_->isArena() &&
|
|
!mem1->getAllowedPeerAccess() && srcDevices.size() == 1) {
|
|
void* src = reinterpret_cast<void*>(mem1->originalDeviceAddress());
|
|
accessAllowed = srcDevices[0]->deviceAllowAccess(src);
|
|
mem1->setAllowedPeerAccess(true);
|
|
}
|
|
|
|
device::Memory* mem2 = memory2_->getDeviceMemory(*dstDevices[0]);
|
|
if (!memory2_->isArena() &&
|
|
!mem2->getAllowedPeerAccess() && dstDevices.size() == 1) {
|
|
void* dst = reinterpret_cast<void*>(mem2->originalDeviceAddress());
|
|
accessAllowed &= dstDevices[0]->deviceAllowAccess(dst);
|
|
mem2->setAllowedPeerAccess(true);
|
|
}
|
|
}
|
|
return accessAllowed;
|
|
}
|
|
|
|
bool TwoMemoryArgsCommand::validateMemory() {
|
|
// Runtime disables deferred memory allocation for single device.
|
|
// Hence ignore memory validations
|
|
if (queue()->context().devices().size() == 1) {
|
|
return true;
|
|
}
|
|
device::Memory* mem = memory1_->getDeviceMemory(queue()->device());
|
|
if (NULL == mem) {
|
|
LogPrintfError("Can't allocate memory size - 0x%08X bytes!", memory1_->getSize());
|
|
return false;
|
|
}
|
|
mem = memory2_->getDeviceMemory(queue()->device());
|
|
if (NULL == mem) {
|
|
LogPrintfError("Can't allocate memory size - 0x%08X bytes!", memory2_->getSize());
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
bool ReadMemoryCommand::isEntireMemory() const {
|
|
return source().isEntirelyCovered(origin(), size());
|
|
}
|
|
|
|
bool WriteMemoryCommand::isEntireMemory() const {
|
|
return destination().isEntirelyCovered(origin(), size());
|
|
}
|
|
|
|
bool SvmMapMemoryCommand::isEntireMemory() const {
|
|
return getSvmMem()->isEntirelyCovered(origin(), size());
|
|
}
|
|
|
|
bool FillMemoryCommand::isEntireMemory() const {
|
|
return memory().isEntirelyCovered(origin(), size());
|
|
}
|
|
|
|
bool CopyMemoryCommand::isEntireMemory() const {
|
|
bool result = false;
|
|
|
|
switch (type()) {
|
|
case CL_COMMAND_COPY_IMAGE_TO_BUFFER: {
|
|
Coord3D imageSize(size()[0] * size()[1] * size()[2] *
|
|
source().asImage()->getImageFormat().getElementSize());
|
|
result = source().isEntirelyCovered(srcOrigin(), size()) &&
|
|
destination().isEntirelyCovered(dstOrigin(), imageSize);
|
|
} break;
|
|
case CL_COMMAND_COPY_BUFFER_TO_IMAGE: {
|
|
Coord3D imageSize(size()[0] * size()[1] * size()[2] *
|
|
destination().asImage()->getImageFormat().getElementSize());
|
|
result = source().isEntirelyCovered(srcOrigin(), imageSize) &&
|
|
destination().isEntirelyCovered(dstOrigin(), size());
|
|
} break;
|
|
case CL_COMMAND_COPY_BUFFER_RECT: {
|
|
Coord3D rectSize(size()[0] * size()[1] * size()[2]);
|
|
Coord3D srcOffs(srcRect().start_);
|
|
Coord3D dstOffs(dstRect().start_);
|
|
result = source().isEntirelyCovered(srcOffs, rectSize) &&
|
|
destination().isEntirelyCovered(dstOffs, rectSize);
|
|
} break;
|
|
default:
|
|
result = source().isEntirelyCovered(srcOrigin(), size()) &&
|
|
destination().isEntirelyCovered(dstOrigin(), size());
|
|
break;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
bool MapMemoryCommand::isEntireMemory() const {
|
|
return memory().isEntirelyCovered(origin(), size());
|
|
}
|
|
|
|
void UnmapMemoryCommand::releaseResources() {
|
|
//! @todo This is a workaround to a deadlock on indirect map release.
|
|
//! Remove this code when CAL will have a refcounter on memory.
|
|
//! decIndMapCount() has to go back to submitUnmapMemory()
|
|
device::Memory* mem = memory_->getDeviceMemory(queue()->device());
|
|
if (NULL != mem) {
|
|
mem->releaseIndirectMap();
|
|
}
|
|
|
|
OneMemoryArgCommand::releaseResources();
|
|
}
|
|
|
|
bool MigrateMemObjectsCommand::validateMemory() {
|
|
// Runtime disables deferred memory allocation for single device.
|
|
// Hence ignore memory validations
|
|
if (queue()->context().devices().size() == 1) {
|
|
return true;
|
|
}
|
|
for (const auto& it : memObjects_) {
|
|
device::Memory* mem = it->getDeviceMemory(queue()->device());
|
|
if (NULL == mem) {
|
|
LogPrintfError("Can't allocate memory size - 0x%08X bytes!", it->getSize());
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// =================================================================================================
|
|
int32_t NDRangeKernelCommand::AllocCaptureSetValidate(void** kernelParams, address kernArgs,
|
|
size_t kernArgsSize) {
|
|
const amd::Device& device = queue()->device();
|
|
// Validate the kernel before submission
|
|
if (!queue()->device().validateKernel(kernel(), queue()->vdev(), cooperativeGroups())) {
|
|
return CL_OUT_OF_RESOURCES;
|
|
}
|
|
|
|
parameters_ = kernel().parameters().alloc(*queue()->vdev());
|
|
if (parameters_ == nullptr) {
|
|
LogError("Cannot allocate memory for parameters_");
|
|
return CL_OUT_OF_RESOURCES;
|
|
}
|
|
|
|
if (!kernel().parameters().captureAndSet(kernelParams, kernArgs, kernArgsSize, parameters_)) {
|
|
LogError("Cannot capture and set the kernel parameters");
|
|
return CL_OUT_OF_RESOURCES;
|
|
}
|
|
return CL_SUCCESS;
|
|
}
|
|
|
|
int32_t NDRangeKernelCommand::captureAndValidate() {
|
|
const amd::Device& device = queue()->device();
|
|
// Validate the kernel before submission
|
|
if (!queue()->device().validateKernel(kernel(), queue()->vdev(), cooperativeGroups())) {
|
|
return CL_OUT_OF_RESOURCES;
|
|
}
|
|
|
|
int32_t error;
|
|
uint64_t lclMemSize = kernel().getDeviceKernel(device)->workGroupInfo()->localMemSize_;
|
|
parameters_ = kernel().parameters().capture(*queue()->vdev(),
|
|
sharedMemBytes_ + lclMemSize, &error);
|
|
return error;
|
|
}
|
|
|
|
bool ExtObjectsCommand::validateMemory() {
|
|
// Always process GL objects, even if deferred allocations are disabled,
|
|
// because processGLResource() calls OGL Acquire().
|
|
bool retVal = true;
|
|
for (const auto& it : memObjects_) {
|
|
device::Memory* mem = it->getDeviceMemory(queue()->device());
|
|
if (NULL == mem) {
|
|
LogPrintfError("Can't allocate memory size - 0x%08X bytes!", it->getSize());
|
|
return false;
|
|
}
|
|
retVal = processGLResource(mem);
|
|
}
|
|
return retVal;
|
|
}
|
|
|
|
bool AcquireExtObjectsCommand::processGLResource(device::Memory* mem) {
|
|
return mem->processGLResource(device::Memory::GLDecompressResource);
|
|
}
|
|
|
|
bool ReleaseExtObjectsCommand::processGLResource(device::Memory* mem) {
|
|
return mem->processGLResource(device::Memory::GLInvalidateFBO);
|
|
}
|
|
|
|
bool MakeBuffersResidentCommand::validateMemory() {
|
|
// Runtime disables deferred memory allocation for single device.
|
|
// Hence ignore memory validations
|
|
if (queue()->context().devices().size() == 1) {
|
|
return true;
|
|
}
|
|
for (const auto& it : memObjects_) {
|
|
device::Memory* mem = it->getDeviceMemory(queue()->device());
|
|
if (NULL == mem) {
|
|
LogPrintfError("Can't allocate memory size - 0x%08X bytes!", it->getSize());
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool ThreadTraceMemObjectsCommand::validateMemory() {
|
|
// Runtime disables deferred memory allocation for single device.
|
|
// Hence ignore memory validations
|
|
if (queue()->context().devices().size() == 1) {
|
|
return true;
|
|
}
|
|
for (auto it = memObjects_.cbegin(); it != memObjects_.cend(); it++) {
|
|
device::Memory* mem = (*it)->getDeviceMemory(queue()->device());
|
|
if (NULL == mem) {
|
|
for (auto tmpIt = memObjects_.cbegin(); tmpIt != it; tmpIt++) {
|
|
device::Memory* tmpMem = (*tmpIt)->getDeviceMemory(queue()->device());
|
|
delete tmpMem;
|
|
}
|
|
LogPrintfError("Can't allocate memory size - 0x%08X bytes!", (*it)->getSize());
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool CopyMemoryP2PCommand::validateMemory() {
|
|
amd::Device* queue_device = &queue()->device();
|
|
|
|
// Rocr backend maps memory from different devices by default and runtime doesn't need to track
|
|
// extra memory objects. Also P2P staging buffer always allocated
|
|
if (queue_device->settings().rocr_backend_) {
|
|
return validatePeerMemory();
|
|
}
|
|
|
|
const std::vector<Device*>& devices = memory1_->getContext().devices();
|
|
if (devices.size() != 1) {
|
|
LogError("Can't allocate memory object for P2P extension");
|
|
return false;
|
|
}
|
|
device::Memory* mem = memory1_->getDeviceMemory(*devices[0]);
|
|
if (nullptr == mem) {
|
|
LogPrintfError("Can't allocate memory size - 0x%08X bytes!", memory1_->getSize());
|
|
return false;
|
|
}
|
|
const std::vector<Device*>& devices2 = memory2_->getContext().devices();
|
|
if (devices2.size() != 1) {
|
|
LogError("Can't allocate memory object for P2P extension");
|
|
return false;
|
|
}
|
|
mem = memory2_->getDeviceMemory(*devices2[0]);
|
|
if (nullptr == mem) {
|
|
LogPrintfError("Can't allocate memory size - 0x%08X bytes!", memory2_->getSize());
|
|
return false;
|
|
}
|
|
bool p2pStaging = false;
|
|
// Validate P2P memories on the current device, if any of them is null, then it's p2p staging
|
|
if ((nullptr == memory1_->getDeviceMemory(queue()->device())) ||
|
|
(nullptr == memory2_->getDeviceMemory(queue()->device()))) {
|
|
p2pStaging = true;
|
|
}
|
|
|
|
if (devices[0]->P2PStage() != nullptr && p2pStaging) {
|
|
amd::ScopedLock lock(devices[0]->P2PStageOps());
|
|
// Make sure runtime allocates memory on every device
|
|
for (uint d = 0; d < devices[0]->GlbCtx().devices().size(); ++d) {
|
|
device::Memory* mem = devices[0]->P2PStage()->getDeviceMemory(*devices[0]->GlbCtx().devices()[d]);
|
|
if (nullptr == mem) {
|
|
DevLogPrintfError("Cannot get P2P stage Device Memory for device: 0x%x \n",
|
|
devices[0]->GlbCtx().devices()[d]);
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// ================================================================================================
|
|
bool SvmPrefetchAsyncCommand::validateMemory() {
|
|
amd::Memory* svmMem = amd::MemObjMap::FindMemObj(dev_ptr());
|
|
if (nullptr == svmMem) {
|
|
LogPrintfError("SvmPrefetchAsync received unknown memory for prefetch: %p!", dev_ptr());
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
} // namespace amd
|