SWDEV-443760 - Enable device kern args

- Implement workaround to ensure HDP writes are done by writing and
reading the HDP MMIO register.
- Implement the same workaround for graphs, we no longer need sentinel
write/readback

Change-Id: I0d3027b46a1f61131ec62e3c8c669ff5184fa6b2


[ROCm/clr commit: f138e0d113]

projects/clr/hipamd/src/hip_graph_internal.cpp

@@ -359,8 +359,6 @@ hipError_t GraphExec::CaptureAQLPackets() {
     auto device = g_devices[ihipGetDevice()]->devices()[0];
     if (kernArgSizeForGraph != 0) {
       if (device->info().largeBar_) {
-        // Pad kernel argument buffer with sentinal size bytes to do a readback later
-        kernArgSizeForGraph += sizeof(int);
         kernarg_pool_graph_ =
             reinterpret_cast<address>(device->deviceLocalAlloc(kernArgSizeForGraph));
         device_kernarg_pool_ = true;
@@ -391,27 +389,13 @@ hipError_t GraphExec::CaptureAQLPackets() {
       }
     }
 
-    if (device_kernarg_pool_) {
-      // Write HDP_MEM_COHERENCY_FLUSH_CNTL reg to initiate flush read to HDP mem. Verify mem
-      // by readback of sentinal value at the tail end of the kernarg surface (allocated above)
-      // This needs to be done for PCIE connected devices only. HDP path is disabled for XGMI
-      // between CPU<->GPU
-      if (!device->isXgmi()) {
-        int host_val = 1;
-        address dev_ptr = kernarg_pool_graph_ + kernarg_pool_size_graph_ - sizeof(int);
-        *(reinterpret_cast<int*>(dev_ptr)) = host_val;
-        if (device->info().hdpMemFlushCntl == nullptr) {
-          amd::Command* command = new amd::Marker(*capture_stream_, true);
-          if (command != nullptr) {
-            command->enqueue();
-            command->release();
-          }
-        } else {
-          *device->info().hdpMemFlushCntl = 1;
-        }
-        if (*(reinterpret_cast<volatile int*>(dev_ptr)) != host_val);
+    if (device_kernarg_pool_ && !device->isXgmi()) {
+      *device->info().hdpMemFlushCntl = 1u;
+      if (*device->info().hdpMemFlushCntl != UINT32_MAX) {
+        LogError("Unexpected HDP Register readback value!");
       }
     }
+
     ResetQueueIndex();
   }
   return status;