SWDEV-301667 - Add cache state for a device

- Add a global cache state for a device to indicate scopes of submitted
AQL packets
- Remove scopes for TS marker if hipEventReleaseToDevice is passed. Set
env ROC_EVENT_NO_FLUSH=1 to use NOP AQL for event records.
It would flush caches by default with system scope release.
- Calling finish() should ensure if caches are flushed, if not queue a
marker

Change-Id: Ibbbdbb1cd7ac61cb35649169212142545be159e0

rocclr/platform/commandqueue.cpp

@@ -106,20 +106,20 @@ bool HostQueue::terminate() {
 
 void HostQueue::finish() {
   Command* command = nullptr;
+  bool isCacheFlushed = device().IsCacheFlushed(Device::CacheState::kCacheStateSystem);
   if (IS_HIP) {
     command = getLastQueuedCommand(true);
-    // Check if the queue has nothing to process and return
-    if (AMD_DIRECT_DISPATCH &&  command == nullptr) {
+    if (AMD_DIRECT_DISPATCH && isCacheFlushed && command == nullptr) {
       return;
     }
   }
-  if (nullptr == command) {
+  if (nullptr == command || !isCacheFlushed) {
     // Send a finish to make sure we finished all commands
     command = new Marker(*this, false);
     if (command == NULL) {
       return;
     }
-    ClPrint(LOG_DEBUG, LOG_CMD, "marker is queued");
+    ClPrint(LOG_DEBUG, LOG_CMD, "Marker queued, Cache Flushed = %d", isCacheFlushed);
     command->enqueue();
   }
   // Check HW status of the ROCcrl event. Note: not all ROCclr modes support HW status
@@ -194,7 +194,7 @@ void HostQueue::loop(device::VirtualDevice* virtualDevice) {
       continue;
     }
 
-    ClPrint(LOG_DEBUG, LOG_CMD, "command (%s) is submitted: %p", getOclCommandKindString(command->type()), command);
+    ClPrint(LOG_DEBUG, LOG_CMD, "Command (%s) submitted: %p", getOclCommandKindString(command->type()), command);
 
     command->setStatus(CL_SUBMITTED);