From 39ef88d37f322d0af5dc197e8a3828950a0b038d Mon Sep 17 00:00:00 2001
From: Yong Zhao <Yong.Zhao@amd.com>
Date: Mon, 15 Oct 2018 18:29:08 -0400
Subject: [PATCH] libhsakmt: Introduce HSA_ZFB environment variable

This variable is 0 by default. When set to 1, it means there is no frame
buffer, so all memory allocation is routed to system memory. This mode
is mainly used during emulation.

Use CoarseGrain for VRAM under ZFB mode

Change-Id: I29e8e98be56935e3ceb94782d70771cc45700749
Signed-off-by: Yong Zhao <Yong.Zhao@amd.com>


[ROCm/ROCR-Runtime commit: 51ee5c324a33c3dbb616ede046fedf683719a6f6]
---
 projects/rocr-runtime/src/memory.c    |  9 ++++++++-
 projects/rocr-runtime/src/openclose.c | 18 +++++++++++++-----
 2 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/projects/rocr-runtime/src/memory.c b/projects/rocr-runtime/src/memory.c
index b9a01c3f0f..2bd67323d8 100644
--- a/projects/rocr-runtime/src/memory.c
+++ b/projects/rocr-runtime/src/memory.c
@@ -34,6 +34,8 @@
 #include <fcntl.h>
 #include "fmm.h"
 
+extern int zfb_support;
+
 HSAKMT_STATUS HSAKMTAPI hsaKmtSetMemoryPolicy(HSAuint32 Node,
 					      HSAuint32 DefaultPolicy,
 					      HSAuint32 AlternatePolicy,
@@ -147,10 +149,15 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemory(HSAuint32 PreferredNode,
 	}
 
 	/* GPU allocated system memory */
-	if (!gpu_id || !MemFlags.ui32.NonPaged) {
+	if (!gpu_id || !MemFlags.ui32.NonPaged || zfb_support) {
 		/* Backwards compatibility hack: Allocate system memory if app
 		 * asks for paged memory from a GPU node.
 		 */
+
+		/* If allocate VRAM under ZFB mode */
+		if (zfb_support && gpu_id && MemFlags.ui32.NonPaged == 1)
+			MemFlags.ui32.CoarseGrain = 1;
+
 		*MemoryAddress = fmm_allocate_host(PreferredNode,  *MemoryAddress,
 						   SizeInBytes,	MemFlags);
 
diff --git a/projects/rocr-runtime/src/openclose.c b/projects/rocr-runtime/src/openclose.c
index 92848f2b91..d4eb264783 100644
--- a/projects/rocr-runtime/src/openclose.c
+++ b/projects/rocr-runtime/src/openclose.c
@@ -39,6 +39,9 @@ static const char kfd_device_name[] = "/dev/kfd";
 static pid_t parent_pid = -1;
 int hsakmt_debug_level;
 
+/* zfb is mainly used during emulation */
+int zfb_support;
+
 static bool is_forked_child(void)
 {
 	pid_t cur_pid = getpid();
@@ -79,14 +82,14 @@ static inline void init_page_size(void)
 	PAGE_SHIFT = ffs(PAGE_SIZE) - 1;
 }
 
-/* Normally libraries don't print messages. For debugging purpose, we'll
- * print messages if an environment variable, HSAKMT_DEBUG_LEVEL, is set.
- */
-static void init_debug_level(void)
+static void init_vars_from_env(void)
 {
 	char *envvar;
 	int debug_level;
 
+	/* Normally libraries don't print messages. For debugging purpose, we'll
+	 * print messages if an environment variable, HSAKMT_DEBUG_LEVEL, is set.
+	 */
 	hsakmt_debug_level = HSAKMT_DEBUG_LEVEL_DEFAULT;
 
 	envvar = getenv("HSAKMT_DEBUG_LEVEL");
@@ -96,6 +99,11 @@ static void init_debug_level(void)
 				debug_level <= HSAKMT_DEBUG_LEVEL_DEBUG)
 			hsakmt_debug_level = debug_level;
 	}
+
+	/* Check whether to support Zero frame buffer */
+	envvar = getenv("HSA_ZFB");
+	if (envvar)
+		zfb_support = atoi(envvar);
 }
 
 HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void)
@@ -114,7 +122,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void)
 		clear_after_fork();
 
 	if (kfd_open_count == 0) {
-		init_debug_level();
+		init_vars_from_env();
 
 		fd = open(kfd_device_name, O_RDWR | O_CLOEXEC);