From 57cbf865cea6d074f121520aa6dc6dacb3b0ac46 Mon Sep 17 00:00:00 2001
From: Sean Keely <Sean.Keely@amd.com>
Date: Fri, 20 Apr 2018 18:09:50 -0500
Subject: [PATCH] Enable large scratch on GFX8.

Ensure system release fence is set on GFX8 large scratch using packets.

Change-Id: I13cfdcd35969482ea6e95e0b352f5cb3a0454b86


[ROCm/ROCR-Runtime commit: 5f25619bb7b9ac14dd37df88ae943b55f8ecb955]
---
 .../runtime/hsa-runtime/core/inc/queue.h            |  3 +--
 .../hsa-runtime/core/runtime/amd_aql_queue.cpp      | 13 +++++++++++--
 .../hsa-runtime/core/runtime/amd_gpu_agent.cpp      |  2 +-
 3 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/queue.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/queue.h
index bf148cc241..811206457e 100644
--- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/queue.h
+++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/queue.h
@@ -75,8 +75,7 @@ struct AqlPacket {
 
   std::string string() const {
     std::stringstream string;
-    uint8_t type = ((dispatch.header >> HSA_PACKET_HEADER_TYPE) &
-                    ((1 << HSA_PACKET_HEADER_WIDTH_TYPE) - 1));
+    uint8_t type = this->type();
 
     const char* type_names[] = {
         "HSA_PACKET_TYPE_VENDOR_SPECIFIC", "HSA_PACKET_TYPE_INVALID",
diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp
index f96c433163..cac25fcf4f 100644
--- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp
+++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp
@@ -746,7 +746,7 @@ bool AqlQueue::DynamicScratchHandler(hsa_signal_value_t error_code, void* arg) {
       uint64_t pkt_slot_idx =
           queue->amd_queue_.read_dispatch_id & (queue->amd_queue_.hsa_queue.size - 1);
 
-      const core::AqlPacket& pkt =
+      core::AqlPacket& pkt =
           ((core::AqlPacket*)queue->amd_queue_.hsa_queue.base_address)[pkt_slot_idx];
 
       uint32_t scratch_request = pkt.dispatch.private_segment_size;
@@ -771,8 +771,17 @@ bool AqlQueue::DynamicScratchHandler(hsa_signal_value_t error_code, void* arg) {
           errorCode = HSA_STATUS_ERROR_OUT_OF_RESOURCES;
         } else {
           // Mark large scratch allocation for single use.
-          if (scratch.large)
+          if (scratch.large) {
             queue->amd_queue_.queue_properties |= AMD_QUEUE_PROPERTIES_USE_SCRATCH_ONCE;
+            // Set system release fence to flush scratch stores with older firmware versions.
+            if ((queue->agent_->isa()->GetMajorVersion() == 8) &&
+                (queue->agent_->GetMicrocodeVersion() < 729)) {
+              pkt.dispatch.header &= ~(((1 << HSA_PACKET_HEADER_WIDTH_SCRELEASE_FENCE_SCOPE) - 1)
+                                       << HSA_PACKET_HEADER_SCRELEASE_FENCE_SCOPE);
+              pkt.dispatch.header |=
+                  (HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_SCRELEASE_FENCE_SCOPE);
+            }
+          }
           // Reset scratch memory related entities for the queue
           queue->InitScratchSRD();
           // Restart the queue.
diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp
index 7d5727e182..3ae2640e4f 100644
--- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp
+++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp
@@ -976,7 +976,7 @@ void GpuAgent::AcquireQueueScratch(ScratchInfo& scratch) {
   ScopedAcquire<KernelMutex> lock(&scratch_lock_);
   bool large = (scratch.size > 6 * 1024 * 1024) ||
       (scratch_pool_.size() - scratch_pool_.remaining() > 24 * 6 * 1024 * 1024);
-  large = (isa_->GetMajorVersion() < 9) ? false : large;
+  large = (isa_->GetMajorVersion() < 8) ? false : large;
   if (large)
     scratch.queue_base = scratch_pool_.alloc_high(scratch.size);
   else