Make host memory allocations coherent by default.

Associated change is to optimize event recording so it uses
agent-scope releaes (since it was only using system-scope release
to support non-coherent host mem).

Flags and environment variables exist to obtain previous behavior
if desired.  Options are documented in new performance guide.

src/hip_device.cpp

@@ -180,6 +180,7 @@ hipError_t hipDeviceReset(void)
     return ihipLogStatus(hipSuccess);
 }
 
+
 hipError_t ihipDeviceSetState(void)
 {
     hipError_t e = hipErrorInvalidContext;

src/hip_hcc.cpp

@@ -83,8 +83,8 @@ int HIP_HIDDEN_FREE_MEM = 256;
 int HIP_FORCE_SYNC_COPY = 0;
 
 // TODO - set these to 0 and 1
-int HIP_EVENT_SYS_RELEASE=1;
-int HIP_COHERENT_HOST_ALLOC = 0;
+int HIP_EVENT_SYS_RELEASE=0;
+int HIP_HOST_COHERENT = 1;
 
 // TODO - set to 0 once we resolve stability.
 // USE_ HIP_SYNC_HOST_ALLOC
@@ -1226,7 +1226,7 @@ void HipReadEnv()
     READ_ENV_I(release, HIP_SYNC_NULL_STREAM, 0, "Synchronize on host for null stream submissions");
 
 
-    READ_ENV_I(release, HIP_COHERENT_HOST_ALLOC, 0, "If set, all host memory will be allocated as fine-grained system memory.  This allows threadfence_system to work but prevents host memory from being cached on GPU which may have performance impact.");
+    READ_ENV_I(release, HIP_HOST_COHERENT, 0, "If set, all host memory will be allocated as fine-grained system memory.  This allows threadfence_system to work but prevents host memory from being cached on GPU which may have performance impact.");
 
 
     READ_ENV_I(release, HCC_OPT_FLUSH, 0, "When set, use agent-scope fence operations rather than system-scope fence operationsflush when possible. This flag controls both HIP and HCC behavior.");

src/hip_hcc_internal.h

@@ -59,7 +59,7 @@ extern int HIP_STREAM_SIGNALS;  /* number of signals to allocate at stream creat
 extern int HIP_VISIBLE_DEVICES; /* Contains a comma-separated sequence of GPU identifiers */
 extern int HIP_FORCE_P2P_HOST;
 
-extern int HIP_COHERENT_HOST_ALLOC;
+extern int HIP_HOST_COHERENT;
 
 extern int HIP_HIDDEN_FREE_MEM;
 //---

src/hip_memory.cpp

@@ -293,7 +293,7 @@ hipError_t hipHostMalloc(void** ptr, size_t sizeBytes, unsigned int flags)
                 amFlags = amHostNonCoherent;
             } else {
                 // depends on env variables:
-                amFlags = HIP_COHERENT_HOST_ALLOC ? amHostCoherent : amHostNonCoherent;
+                amFlags = HIP_HOST_COHERENT ? amHostCoherent : amHostNonCoherent;
             }