In hipFree, synchronize owner of memory (#1018)

* In hipFree, if memory is associated with a device, synchronize that device's streams.

This changes the behavior from synchronizing the currently set TLS device.

* All devices sync in hipFree for _appId=-1 case.

* Revert "All devices sync in hipFree for _appId=-1 case."

This reverts commit 1efb34d6a8426661e45bc5f763422a1147aeac10.

* add HIP_SYNC_FREE env var


[ROCm/clr commit: cf8fb43e6b]

projects/clr/hipamd/src/hip_memory.cpp

@@ -1902,10 +1902,6 @@ hipError_t hipFree(void* ptr) {
 
     hipError_t hipStatus = hipErrorInvalidDevicePointer;
 
-    // Synchronize to ensure all work has finished.
-    ihipGetTlsDefaultCtx()->locked_waitAllStreams();  // ignores non-blocking streams, this waits
-                                                      // for all activity to finish.
-
     if (ptr) {
         hc::accelerator acc;
 #if (__hcc_workweek__ >= 17332)
@@ -1916,6 +1912,29 @@ hipError_t hipFree(void* ptr) {
         am_status_t status = hc::am_memtracker_getinfo(&amPointerInfo, ptr);
         if (status == AM_SUCCESS) {
             if (amPointerInfo._hostPointer == NULL) {
+                if (HIP_SYNC_FREE) {
+                    // Synchronize all devices, all streams
+                    // to ensure all work has finished on all devices.
+                    // This is disabled by default.
+                    for (unsigned i = 0; i < g_deviceCnt; i++) {
+                        ihipGetPrimaryCtx(i)->locked_waitAllStreams();
+                    }
+                }
+                else {
+                    ihipCtx_t* ctx;
+                    if (amPointerInfo._appId != -1) {
+#if USE_APP_PTR_FOR_CTX
+                        ctx = static_cast<ihipCtx_t*>(amPointerInfo._appPtr);
+#else
+                        ctx = ihipGetPrimaryCtx(amPointerInfo._appId);
+#endif
+                    } else {
+                        ctx = ihipGetTlsDefaultCtx();
+                    }
+                    // Synchronize to ensure all work has finished on device owning the memory.
+                    ctx->locked_waitAllStreams();  // ignores non-blocking streams, this waits
+                                                   // for all activity to finish.
+                }
                 hc::am_free(ptr);
                 hipStatus = hipSuccess;
             }