Re-sync with upstream.

[ROCm/hip commit: baf50a5311]

projects/hip/src/hip_hcc.cpp

@@ -37,6 +37,7 @@ THE SOFTWARE.
 #include <vector>
 #include <algorithm>
 #include <atomic>
+#include <mutex>
 
 #include <hc.hpp>
 #include <hc_am.hpp>
@@ -1409,9 +1410,38 @@ void ihipInit()
     tprintf(DB_SYNC, "pid=%u %-30s g_numLogicalThreads=%u\n", getpid(), "<ihipInit>", g_numLogicalThreads);
 }
 
+hipError_t ihipStreamSynchronize(hipStream_t stream)
+{
+    hipError_t e = hipSuccess;
 
+    if (stream == hipStreamNull) {
+        ihipCtx_t *ctx = ihipGetTlsDefaultCtx();
+        ctx->locked_syncDefaultStream(true/*waitOnSelf*/, true/*syncToHost*/);
+    } else {
+        // note this does not synchornize with the NULL stream:
+        stream->locked_wait();
+        e = hipSuccess;
+    }
 
+    return e;
+}
 
+void ihipStreamCallbackHandler(ihipStreamCallback_t *cb)
+{
+    hipError_t e = hipSuccess;
+
+    // Notify hipStreamAddCallback that callback handler thread is active
+    std::lock_guard<std::mutex> guard(cb->_mtx);
+    cb->_ready = true;
+
+    // Synchronize stream
+    tprintf(DB_SYNC, "ihipStreamCallbackHandler wait on stream %s\n", ToString(cb->_stream).c_str());
+    e = ihipStreamSynchronize(cb->_stream);
+
+    // Call registered callback function
+    cb->_callback(cb->_stream, e, cb->_userData);
+    delete cb;
+}
 
 //---
 // Get the stream to use for a command submission.

projects/hip/src/hip_hcc_internal.h

@@ -622,6 +622,24 @@ private: // Data
 };
 
 
+//----
+// Internal structure for stream callback handler
+class ihipStreamCallback_t {
+public:
+    ihipStreamCallback_t(hipStream_t stream, hipStreamCallback_t callback, void *userData) :
+        _stream(stream),
+        _callback(callback),
+        _userData(userData)
+        {
+            _ready = false;
+        };
+    hipStream_t                 _stream;
+    hipStreamCallback_t         _callback;
+    void*                       _userData;
+    std::mutex                  _mtx;
+    bool                        _ready;
+};
+
 
 //----
 // Internal event structure:
@@ -931,6 +949,8 @@ ihipCtx_t * ihipGetPrimaryCtx(unsigned deviceIndex);
 
 
 hipStream_t ihipSyncAndResolveStream(hipStream_t);
+hipError_t ihipStreamSynchronize(hipStream_t stream);
+void ihipStreamCallbackHandler(ihipStreamCallback_t *cb);
 
 // Stream printf functions:
 inline std::ostream& operator<<(std::ostream& os, const ihipStream_t& s)

projects/hip/src/hip_memory.cpp

@@ -453,6 +453,7 @@ hipError_t hipArrayCreate ( hipArray** array, const HIP_ARRAY_DESCRIPTOR* pAlloc
         array[0]->width = pAllocateArray->width;
         array[0]->height = pAllocateArray->height;
         array[0]->isDrv = true;
+        array[0]->textureType = hipTextureType2D;
         void ** ptr = &array[0]->data;
 	    if (ctx) {
             const unsigned am_flags = 0;
@@ -1411,6 +1412,65 @@ hipError_t hipMemcpyToArray(hipArray* dst, size_t wOffset, size_t hOffset,
     return ihipLogStatus(e);
 }
 
+hipError_t hipMemcpyFromArray(void* dst, hipArray_const_t srcArray, size_t wOffset, size_t hOffset,
+         size_t count, hipMemcpyKind kind) {
+
+    HIP_INIT_SPECIAL_API((TRACE_MCMD), dst, srcArray, wOffset, hOffset, count, kind);
+
+    hipStream_t stream = ihipSyncAndResolveStream(hipStreamNull);
+
+    hc::completion_future marker;
+
+    hipError_t e = hipSuccess;
+
+    try {
+        stream->locked_copySync((char *)dst, (char*)srcArray->data + wOffset, count, kind);
+    }
+    catch (ihipException &ex) {
+        e = ex._code;
+    }
+
+    return ihipLogStatus(e);
+}
+
+hipError_t hipMemcpyHtoA(hipArray* dstArray, size_t dstOffset, const void* srcHost, size_t count)
+{
+    HIP_INIT_SPECIAL_API((TRACE_MCMD), dstArray, dstOffset, srcHost, count);
+
+    hipStream_t stream = ihipSyncAndResolveStream(hipStreamNull);
+
+    hc::completion_future marker;
+
+    hipError_t e = hipSuccess;
+    try {
+        stream->locked_copySync((char *)dstArray->data + dstOffset, srcHost, count, hipMemcpyHostToDevice);
+    } catch (ihipException &ex) {
+        e = ex._code;
+    }
+
+    return ihipLogStatus(e);
+}
+
+hipError_t hipMemcpyAtoH(void* dst, hipArray* srcArray, size_t srcOffset, size_t count)
+{
+    HIP_INIT_SPECIAL_API((TRACE_MCMD), dst, srcArray, srcOffset, count);
+
+    hipStream_t stream = ihipSyncAndResolveStream(hipStreamNull);
+
+    hc::completion_future marker;
+
+    hipError_t e = hipSuccess;
+
+    try {
+        stream->locked_copySync((char *)dst, (char*)srcArray->data + srcOffset, count, hipMemcpyDeviceToHost);
+    }
+    catch (ihipException &ex) {
+        e = ex._code;
+    }
+
+    return ihipLogStatus(e);
+}
+
 hipError_t hipMemcpy3D(const struct hipMemcpy3DParms *p)
 {
     HIP_INIT_SPECIAL_API((TRACE_MCMD), p);

projects/hip/src/hip_module.cpp

@@ -594,7 +594,6 @@ hipError_t hipModuleGetTexRef(
     const auto it = globals().find(name);
     if (it == globals().end()) return ihipLogStatus(hipErrorInvalidValue);
 
-    *texRef = static_cast<textureReference*>(it->second.get());
-
+    *texRef = reinterpret_cast<textureReference*>(it->second);
     return ihipLogStatus(hipSuccess);
 }

projects/hip/src/hip_stream.cpp

@@ -20,6 +20,8 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
+#include <thread>
+#include <mutex>
 #include "hip/hip_runtime.h"
 #include "hip_hcc_internal.h"
 #include "trace_helper.h"
@@ -147,20 +149,8 @@ hipError_t hipStreamSynchronize(hipStream_t stream)
 {
     HIP_INIT_SPECIAL_API(TRACE_SYNC, stream);
 
-    hipError_t e = hipSuccess;
-
-    if (stream == hipStreamNull) {
-        ihipCtx_t *ctx = ihipGetTlsDefaultCtx();
-        ctx->locked_syncDefaultStream(true/*waitOnSelf*/, true/*syncToHost*/);
-    } else {
-		// note this does not synchornize with the NULL stream:
-        stream->locked_wait();
-        e = hipSuccess;
-    }
-
-
-    return ihipLogStatus(e);
-};
+    return ihipLogStatus(ihipStreamSynchronize(stream));
+}
 
 
 //---
@@ -216,8 +206,20 @@ hipError_t hipStreamAddCallback(hipStream_t stream, hipStreamCallback_t callback
 {
     HIP_INIT_API(stream, callback, userData, flags);
     hipError_t e = hipSuccess;
-    //--- explicitly synchronize stream to add callback routines
-    hipStreamSynchronize(stream);
-    callback(stream, e, userData);
+
+    // Create a thread in detached mode to handle callback
+    ihipStreamCallback_t *cb = new ihipStreamCallback_t(stream, callback, userData);
+    std::thread (ihipStreamCallbackHandler, cb).detach();
+
+    // Wait for thread to be ready
+    cb->_mtx.lock();
+    while(cb->_ready != true)
+    {
+        cb->_mtx.unlock();
+        std::this_thread::sleep_for(std::chrono::milliseconds(10));
+        cb->_mtx.lock();
+    }
+    cb->_mtx.unlock();
+
     return ihipLogStatus(e);
 }

projects/hip/src/program_state.cpp

@@ -151,7 +151,7 @@ namespace
             lock_guard<mutex> lck{mtx};
 
             if (globals().find(x) != globals().cend()) return;
-
+            globals().emplace(x, (void*)(it1->second.first));
             void* p = nullptr;
             hsa_amd_memory_lock(
                 reinterpret_cast<void*>(it1->second.first),
@@ -163,7 +163,6 @@ namespace
             hsa_executable_agent_global_variable_define(
                 executable, agent, x.c_str(), p);
 
-            globals().emplace(x, RAII_global{p, hsa_amd_memory_unlock});
         }
     }
 
@@ -444,9 +443,9 @@ namespace hip_impl
         return r;
     }
 
-    unordered_map<string, RAII_global>& globals()
+    unordered_map<string, void*>& globals()
     {
-        static unordered_map<string, RAII_global> r;
+        static unordered_map<string, void*> r;
         static once_flag f;
         call_once(f, []() { r.reserve(symbol_addresses().size()); });
 
@@ -473,4 +472,4 @@ namespace hip_impl
 
         return executable;
     }
-} // Namespace hip_impl.
+} // Namespace hip_impl.