AI/rocm-systems
2
0
Forka 0
Codice Problemi Pull Request Actions Pacchetti Progetti Rilasci Wiki Attività

P4 to Git Change 1394146 by gandryey@gera-w8 on 2017/04/04 11:59:55

Sfoglia il codice sorgente 
SWDEV-79445 - OCL generic changes and code clean-up
	- Added a simple hang analysis (PAL path only). GPU_ANALYZE_HANG controls the feature. It will create ocl_hang_dump.txt, which will contain OCL resident resources used in the last submissions on this queue and last kernel name. It’s recommended to use this feature with GPU_FLUSH_ON_EXECUTION or AMD_OCL_WAIT_COMMAND

Affected files ...

... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palvirtual.cpp#46 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palvirtual.hpp#20 edit
... //depot/stg/opencl/drivers/opencl/runtime/utils/flags.hpp#270 edit
This commit is contained in:
foreman
2017-04-04 11:09:12 -05:00
parent f5115dca59
commit 738dcbcdd4
3 ha cambiato i file con 89 aggiunte e 18 eliminazioni
Scarica il file Patch Scarica il file Diff Expand all files Collapse all files
rocclr/runtime/device/pal/palvirtual.cpp
+60 -4
Vedi File
@@ -360,6 +360,57 @@ VirtualGPU::Queue::isDone(uint id)
return true;
}
void
VirtualGPU::Queue::DumpMemoryReferences() const
{
std::fstream dump;
std::stringstream file_name("ocl_hang_dump.txt");
uint64_t start = amd::Os::timeNanos() / 1e9;
dump.open(file_name.str().c_str(), (std::fstream::out | std::fstream::app));
// Check if we have OpenCL program
if (dump.is_open()) {
dump << start << " Queue: ";
switch (iQueue_->Type()) {
case Pal::QueueTypeCompute:
dump << "Compute";
break;
case Pal::QueueTypeDma:
dump << "SDMA";
break;
default:
dump << "unknown";
break;
}
dump << "\n" << "Resident memory resources:\n";
uint idx = 0;
for (auto it : memReferences_) {
dump << " " << idx << "\t[";
dump.setf(std::ios::hex, std::ios::basefield);
dump.setf(std::ios::showbase);
dump << (it.first)->Desc().gpuVirtAddr << ", " <<
(it.first)->Desc().gpuVirtAddr + (it.first)->Desc().size;
dump.setf(std::ios::dec);
dump << "] CbId:" << it.second << "\n";
idx++;
}
}
if (last_kernel_ != nullptr) {
const amd::KernelSignature& signature = last_kernel_->signature();
const amd::KernelParameters& params = last_kernel_->parameters();
dump << last_kernel_->name() << std::endl;
for (size_t i = 0; i < signature.numParameters(); ++i) {
const amd::KernelParameterDescriptor& desc = signature.at(i);
// Find if the current argument is a memory object
if ((desc.type_ == T_POINTER) &&
(desc.addressQualifier_ != CL_KERNEL_ARG_ADDRESS_LOCAL)) {
dump << " " << desc.name_ << ": " << std::endl;
}
}
}
dump.close();
}
bool
VirtualGPU::MemoryDependency::create(size_t numMemObj)
{
@@ -1998,6 +2049,8 @@ VirtualGPU::submitKernelInternal(
VirtualGPU* gpuDefQueue = nullptr;
amd::HwDebugManager * dbgManager = dev().hwDebugMgr();
AddKernel(kernel);
// Get the HSA kernel object
const HSAILKernel& hsaKernel =
static_cast<const HSAILKernel&>(*(kernel.getDeviceKernel(dev())));
@@ -3177,18 +3230,21 @@ VirtualGPU::profilingCollectResults(CommandBatch* cb, const amd::Event* waitingE
return found;
}
bool
void
VirtualGPU::addVmMemory(const Memory* memory)
{
queues_[MainEngine]->addCmdMemRef(memory->iMem());
return true;
}
void
VirtualGPU::AddKernel(const amd::Kernel& kernel) const
{
queues_[MainEngine]->last_kernel_ = &kernel;
}
bool
void
VirtualGPU::addDoppRef(const Memory* memory, bool lastDoppCmd, bool pfpaDoppCmd)
{
queues_[MainEngine]->addCmdDoppRef(memory->iMem(), lastDoppCmd, pfpaDoppCmd);
return true;
}
void
rocclr/runtime/device/pal/palvirtual.hpp
+27 -14
Vedi File
@@ -50,6 +50,9 @@ public:
static const uint64_t WaitTimeoutInNsec = 6000000000;
static const uint64_t PollIntervalInNsec = 200000;
Queue(const Queue&) = delete;
Queue& operator=(const Queue&) = delete;
static Queue* Create(
Pal::IDevice* palDev, //!< PAL device object
Pal::QueueType queueType, //!< PAL queue type
@@ -60,13 +63,13 @@ public:
);
Queue(Pal::IDevice* palDev)
: iQueue_(NULL), iDev_(palDev),
: iQueue_(nullptr), last_kernel_(nullptr), iDev_(palDev),
cmdBufIdSlot_(StartCmdBufIdx), cmdBufIdCurrent_(StartCmdBufIdx),
cmbBufIdRetired_(0), cmdCnt_(0), vlAlloc_(64 * Ki)
{
for (uint i = 0; i < MaxCmdBuffers; ++i) {
iCmdBuffs_[i] = NULL;
iCmdFences_[i] = NULL;
iCmdBuffs_[i] = nullptr;
iCmdFences_[i] = nullptr;
}
vlAlloc_.Init();
}
@@ -82,12 +85,12 @@ public:
{
Pal::GpuMemoryRef memRef = {};
memRef.pGpuMemory = iMem;
iDev_->AddGpuMemoryReferences(1, &memRef, NULL,
iDev_->AddGpuMemoryReferences(1, &memRef, nullptr,
Pal::GpuMemoryRefCantTrim);
}
void removeMemRef(Pal::IGpuMemory* iMem) const
{
iDev_->RemoveGpuMemoryReferences(1, &iMem, NULL);
iDev_->RemoveGpuMemoryReferences(1, &iMem, nullptr);
}
// ibReuse forces event wait without polling, to make sure event occured
@@ -112,6 +115,9 @@ public:
else if ((Pal::Result::NotReady == result) ||
(Pal::Result::Timeout == result)) {
LogWarning("PAL fence isn't ready!");
if (GPU_ANALYZE_HANG) {
DumpMemoryReferences();
}
}
else {
LogError("PAL wait for a fence failed!");
@@ -134,10 +140,12 @@ public:
Pal::ICmdBuffer* iCmd() const { return iCmdBuffs_[cmdBufIdSlot_]; }
Pal::IQueue* iQueue_; //!< PAL queue object
Pal::ICmdBuffer* iCmdBuffs_[MaxCmdBuffers]; //!< PAL command buffers
Pal::IFence* iCmdFences_[MaxCmdBuffers]; //!< PAL fences, associated with CMD
Pal::ICmdBuffer* iCmdBuffs_[MaxCmdBuffers]; //!< PAL command buffers
Pal::IFence* iCmdFences_[MaxCmdBuffers];//!< PAL fences, associated with CMD
const amd::Kernel* last_kernel_; //!< Last submitted kernel
private:
void DumpMemoryReferences() const;
Pal::IDevice* iDev_; //!< PAL device
uint cmdBufIdSlot_; //!< Command buffer ID slot for submissions
uint cmdBufIdCurrent_; //!< Current global command buffer ID
@@ -207,7 +215,7 @@ public:
public:
//! Default constructor
MemoryDependency()
: memObjectsInQueue_(NULL)
: memObjectsInQueue_(nullptr)
, numMemObjectsInQueue_(0)
, maxMemObjectsInQueue_(0) {}
@@ -294,7 +302,7 @@ public:
const amd::Kernel& kernel, //!< Kernel for execution
const_address parameters, //!< Parameters for the kernel
bool nativeMem = true, //!< Native memory objects
amd::Event* enqueueEvent = NULL //!< Event provided in the enqueue kernel command
amd::Event* enqueueEvent = nullptr //!< Event provided in the enqueue kernel command
);
void submitNativeFn(amd::NativeFnCommand& vcmd);
void submitFillMemory(amd::FillMemoryCommand& vcmd);
@@ -316,7 +324,7 @@ public:
void releaseMemory(Pal::IGpuMemory* iMem, bool wait = true);
void flush(amd::Command* list = NULL, bool wait = false);
void flush(amd::Command* list = nullptr, bool wait = false);
bool terminate() { return true; }
//! Returns GPU device object associated with this kernel
@@ -350,7 +358,7 @@ public:
//! Wait for all engines on this Virtual GPU
//! Returns TRUE if CPU didn't wait for GPU
bool waitAllEngines(
CommandBatch* cb = NULL //!< Command batch
CommandBatch* cb = nullptr //!< Command batch
);
//! Waits for the latest GPU event with a lock to prevent multiple entries
@@ -382,12 +390,17 @@ public:
);
//! Adds a memory handle into the GSL memory array for Virtual Heap
bool addVmMemory(
void addVmMemory(
const Memory* memory //!< GPU memory object
);
//! Adds the last submitted kernel to the queue for tracking a possible hang
void AddKernel(
const amd::Kernel& kernel //!< AMD kernel object
) const;
//! Adds a dopp desktop texture reference
bool addDoppRef(
void addDoppRef(
const Memory* memory, //!< GPU memory object
bool lastDoopCmd, //!< is the last submission for the pre-present primary
bool pfpaDoppCmd //!< is a submission for the pre-present primary
@@ -565,7 +578,7 @@ private:
//! Awaits a command batch with a waiting event
bool awaitCompletion(
CommandBatch* cb, //!< Command batch for to wait
const amd::Event* waitingEvent = NULL //!< A waiting event
const amd::Event* waitingEvent = nullptr //!< A waiting event
);
//! Detects memory dependency for HSAIL kernels and flushes caches
rocclr/runtime/utils/flags.hpp
+2
Vedi File
@@ -183,6 +183,8 @@ release(bool, GPU_WAVE_LIMIT_ENABLE, false, \
"1 = Enable adaptive wave limiter") \
release(bool, OCL_STUB_PROGRAMS, false, \
"1 = Enables OCL programs stubing") \
release(bool, GPU_ANALYZE_HANG, false, \
"1 = Enables GPU hang analysis") \
release_on_stg(uint, GPU_WAVE_LIMIT_CU_PER_SH, 0, \
"Assume the number of CU per SH for wave limiter") \
release_on_stg(uint, GPU_WAVE_LIMIT_MAX_WAVE, 10, \