SWDEV-459826 - Add a crash dump for a failed queue

The logic can analyze the AQL queue state and
find a failed AQL packet with the kernel's name

Change-Id: I1a478fa2c25462cd07a194784958bdf22454b897

rocclr/device/rocm/rocvirtual.cpp

@@ -891,6 +891,73 @@ static inline void packet_store_release(uint32_t* packet, uint16_t header, uint1
   __atomic_store_n(packet, header | (rest << 16), __ATOMIC_RELEASE);
 }
 
+// ================================================================================================
+void VirtualGPU::AnalyzeAqlQueue() const {
+  const uint32_t queueSize = gpu_queue_->size;
+  const uint32_t queueMask = queueSize - 1;
+  const uint32_t sw_queue_size = queueMask;
+  uint64_t index = hsa_queue_load_write_index_relaxed(gpu_queue_);
+  uint64_t read = hsa_queue_load_read_index_relaxed(gpu_queue_);
+  if (index > read) {
+    int valid_packet_idx = 0;
+    constexpr int kAqlSearchWindow = 32;
+    while (valid_packet_idx < kAqlSearchWindow) {
+      // Read AQL packet header and check if it's invalid, which means it's done
+      auto aql_loc = &(reinterpret_cast<hsa_kernel_dispatch_packet_t*>
+        (gpu_queue_->base_address))[(read + valid_packet_idx) & queueMask];
+      // If the packet is invalid, then continue search
+      if (extractAqlBits((*aql_loc).header, HSA_PACKET_HEADER_TYPE,
+        HSA_PACKET_HEADER_WIDTH_TYPE) == HSA_PACKET_TYPE_INVALID) {
+        valid_packet_idx++;
+      } else {
+        break;
+      }
+    }
+    if (valid_packet_idx == kAqlSearchWindow) {
+      printf("VGPU(%p) Queue(%p). Couldn't find the hang AQL packet!\n", this, gpu_queue_);
+      return;
+    }
+    // Read AQL packet and check if it's a kernel dispatch
+    auto aql_loc = &(reinterpret_cast<hsa_kernel_dispatch_packet_t*>
+      (gpu_queue_->base_address))[(read + valid_packet_idx) & queueMask];
+    auto packet = *aql_loc;
+    auto header = packet.header;
+    if (extractAqlBits(header, HSA_PACKET_HEADER_TYPE, HSA_PACKET_HEADER_WIDTH_TYPE) ==
+        HSA_PACKET_TYPE_KERNEL_DISPATCH) {
+      auto it = dev().KernelMap().find(packet.kernel_object);
+      if (it != dev().KernelMap().end()) {
+        // @note: It's possible to demangle the name with comgr
+        printf("Kernel Name: %s\n", it->second.name().c_str());
+      } else {
+        printf("VGPU(%p) Queue(%p). Couldn't find kernel\n", this, gpu_queue_);
+      }
+      printf("VGPU=%p SWq=%p, HWq=%p, id=%ld\n\tDispatch Header = "
+             "0x%x (type=%d, barrier=%d, acquire=%d, release=%d), "
+             "setup=%d\n\tgrid=[%u, %u, %u], workgroup=[%u, %u, %u]\n\tprivate_seg_size=%u, "
+             "group_seg_size=%u\n\tkernel_obj=0x%lx, "
+             "kernarg_address=0x%p\n\tcompletion_signal=0x%lx, "
+             "correlation_id=%lu\n\trptr=%lu, wptr=%lu\n",
+             this, gpu_queue_, gpu_queue_->base_address, gpu_queue_->id, header,
+             extractAqlBits(header, HSA_PACKET_HEADER_TYPE, HSA_PACKET_HEADER_WIDTH_TYPE),
+             extractAqlBits(header, HSA_PACKET_HEADER_BARRIER, HSA_PACKET_HEADER_WIDTH_BARRIER),
+             extractAqlBits(header, HSA_PACKET_HEADER_SCACQUIRE_FENCE_SCOPE,
+                            HSA_PACKET_HEADER_WIDTH_SCACQUIRE_FENCE_SCOPE),
+             extractAqlBits(header, HSA_PACKET_HEADER_SCRELEASE_FENCE_SCOPE,
+                            HSA_PACKET_HEADER_WIDTH_SCRELEASE_FENCE_SCOPE), 0,
+             packet.grid_size_x, packet.grid_size_y, packet.grid_size_z,
+             packet.workgroup_size_x, packet. workgroup_size_y, packet.workgroup_size_z,
+             packet.private_segment_size, packet.group_segment_size, packet.kernel_object,
+             packet.kernarg_address, packet.completion_signal.handle, packet.reserved2,
+             read, index);
+    } else {
+      printf("VGPU(%p) Queue(%p) rptr=%lu, wptr=%lu. A barrier packet in the queue!\n",
+             this, gpu_queue_, read, index);
+    }
+  } else {
+    printf("VGPU(%p) Queue(%p) is idle\n", this, gpu_queue_);
+  }
+}
+
 // ================================================================================================
 template <typename AqlPacket>
 bool VirtualGPU::dispatchGenericAqlPacket(
@@ -2907,12 +2974,13 @@ void VirtualGPU::submitMigrateMemObjects(amd::MigrateMemObjectsCommand& vcmd) {
 // ================================================================================================
 static void callbackQueue(hsa_status_t status, hsa_queue_t* queue, void* data) {
   if (status != HSA_STATUS_SUCCESS && status != HSA_STATUS_INFO_BREAK) {
+    VirtualGPU* vgpu = reinterpret_cast<VirtualGPU*>(data);
+    vgpu->AnalyzeAqlQueue();
     // Abort on device exceptions.
     const char* errorMsg = 0;
     hsa_status_string(status, &errorMsg);
     if (status == HSA_STATUS_ERROR_OUT_OF_RESOURCES) {
       size_t global_available_mem = 0;
-      VirtualGPU* vgpu = reinterpret_cast<VirtualGPU*>(data);
       if (HSA_STATUS_SUCCESS != hsa_agent_get_info(vgpu->gpu_device(),
                          static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_MEMORY_AVAIL),
                          &global_available_mem)) {
@@ -3482,7 +3550,7 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes,
     dispatchPacket.header = kInvalidAql;
     dispatchPacket.kernel_object = gpuKernel.KernelCodeHandle();
 
-   // dispatchPacket.header = aqlHeader_;
+    // dispatchPacket.header = aqlHeader_;
     // dispatchPacket.setup |= sizes.dimensions() << HSA_KERNEL_DISPATCH_PACKET_SETUP_DIMENSIONS;
     dispatchPacket.grid_size_x = sizes.dimensions() > 0 ? newGlobalSize[0] : 1;
     dispatchPacket.grid_size_y = sizes.dimensions() > 1 ? newGlobalSize[1] : 1;