Merge branch 'privatestaging' of https://github.com/AMDComputeLibraries/HIP-privatestaging into privatestaging

[ROCm/hip commit: bdd26bd1f1]

projects/hip/include/hcc_detail/hip_hcc.h

@@ -35,6 +35,14 @@ THE SOFTWARE.
 //Use the new HCC accelerator_view::copy instead of am_copy
 #define USE_AV_COPY 0
 
+// Compile peer-to-peer support.
+// >= 2 : use HCC hc:accelerator::get_is_peer
+// >= 3 : use hc::am_memtracker_update_peers(...)
+#define USE_PEER_TO_PEER 0
+
+// Use new lock API in HCC:
+#define USE_HCC_LOCK 0
+
 //#define INLINE static inline
 
 //---
@@ -494,11 +502,23 @@ struct ihipEvent_t {
 // will lock the mutex on construction and unlock on destruction.
 //
 // MUTEX_TYPE is template argument so can easily convert to FakeMutex for performance or stress testing.
-template <typename MUTEX_TYPE>
+template <class MUTEX_TYPE>
 class ihipDeviceCriticalBase_t : LockedBase<MUTEX_TYPE>
 {
 public:
-    ihipDeviceCriticalBase_t() :  _stream_id(0) {};
+    ihipDeviceCriticalBase_t() :  _stream_id(0), _peerAgents(nullptr) {};
+   
+    void init(unsigned deviceCnt) {
+        assert(_peerAgents == nullptr);
+        _peerAgents = new hsa_agent_t[deviceCnt];
+    };
+
+    ~ihipDeviceCriticalBase_t()  {
+        if (_peerAgents != nullptr) {
+            delete _peerAgents;
+            _peerAgents = nullptr;
+        }
+    }
     friend class LockedAccessor<ihipDeviceCriticalBase_t>;
 
     std::list<ihipStream_t*> &streams() { return _streams; };
@@ -507,10 +527,24 @@ public:
     // "Allocate" a stream ID:
     ihipStream_t::SeqNum_t  incStreamId() { return _stream_id++; };
 
+    bool addPeer(ihipDevice_t *peer);
+    bool removePeer(ihipDevice_t *peer);
+    void resetPeers(ihipDevice_t *thisDevice);
+
+    uint32_t peerCnt() const { return _peerCnt; };
+    hsa_agent_t *peerAgents() const { return _peerAgents; };
+
 
 private:
     std::list<ihipStream_t*> _streams;   // streams associated with this device.
     ihipStream_t::SeqNum_t   _stream_id;
+
+    // These reflect the currently Enabled set of peers for this GPU:
+    std::list<ihipDevice_t*>  _peers;     // list of enabled peer devices.
+    uint32_t                  _peerCnt;     // number of enabled peers
+    hsa_agent_t              *_peerAgents;  // efficient packed array of enabled agents (to use for allocations.)
+private:
+    void recomputePeerAgents();
 };
 
 // Note Mutex selected based on DeviceMutex
@@ -530,7 +564,7 @@ class ihipDevice_t
 {
 public: // Functions:
     ihipDevice_t() {}; // note: calls constructor for _criticalData 
-    void init(unsigned device_index, hc::accelerator &acc, unsigned flags);
+    void init(unsigned device_index, unsigned deviceCnt, hc::accelerator &acc, unsigned flags);
     ~ihipDevice_t();
 
     void locked_addStream(ihipStream_t *s);
@@ -539,6 +573,8 @@ public: // Functions:
     void locked_waitAllStreams();
     void locked_syncDefaultStream(bool waitOnSelf);
 
+    ihipDeviceCritical_t  &criticalData() { return _criticalData; }; // TODO, move private.  Fix P2P.
+
 public: // Data, set at initialization:
     unsigned                _device_index; // index into g_devices.
 

projects/hip/include/hcc_detail/hip_runtime.h

@@ -24,7 +24,9 @@ THE SOFTWARE.
  *  @brief Contains definitions of APIs for HIP runtime.
  */
 
-#pragma once
+//#pragma once
+#ifndef HIP_RUNTIME_H
+#define HIP_RUNTIME_H
 
 //---
 // Top part of file can be compiled with any compiler
@@ -574,4 +576,4 @@ do {\
  */
 
 
-
+#endif

projects/hip/include/hcc_detail/hip_runtime_api.h

@@ -19,8 +19,9 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
-#pragma once
-
+//#pragma once
+#ifndef HIP_RUNTIME_API_H
+#define HIP_RUNTIME_API_H
 /**
  *  @file  hcc_detail/hip_runtime_api.h
  *  @brief Contains C function APIs for HIP runtime. This file does not use any HCC builtin or special language extensions (-hc mode) ; those functions in hip_runtime.h.
@@ -907,46 +908,58 @@ hipError_t hipMemGetInfo  (size_t * free, size_t * total)   ;
  *
  * Returns "1" in @p canAccessPeer if the specified @p device is capable
  * of directly accessing memory physically located on peerDevice , or "0" if not.
+ *
+ * Returns "0" in @p canAccessPeer if deviceId == peerDeviceId, and both are valid devices : a device is not a peer of itself.
+ *
+ *
+ *
+ * @returns #hipSuccess, 
+ * @returns #hipErrorInvalidDevice if deviceId or peerDeviceId are not valid devices
  */
-hipError_t hipDeviceCanAccessPeer ( int* canAccessPeer, int  device, int  peerDevice );
-
+hipError_t hipDeviceCanAccessPeer (int* canAccessPeer, int deviceId, int peerDeviceId);
 
 
 /**
- * @brief Disables registering memory on peerDevice for direct access from the current device.
+ * @brief Enable direct access from current device's virtual address space to memory allocations physically located on a peer device.  
  *
- * If there are any allocations on peerDevice which were registered in the current device using hipPeerRegister() then these allocations will be automatically unregistered.
- * Returns hipErrorPeerAccessNotEnabled if direct access to memory on peerDevice has not yet been enabled from the current device.
+ * Memory which already allocated on peer device will be mapped into the address space of the current device.  In addition, all
+ * future memory allocations on peerDeviceId will be mapped into the address space of the current device when the memory is allocated.
+ * The peer memory remains accessible from the current device until a call to hipDeviceDisablePeerAccess or hipDeviceReset.
  *
- * @param [in] peerDevice
- * TODO:cudaErrorPeerAccessNotEnabled and cudaErrorInvalidDevice error not supported in HIP, return hipErrorUnknown
- * Returns #hipSuccess, #hipErrorUnknown
- */
-hipError_t  hipDeviceDisablePeerAccess ( int  peerDevice );
-
-/**
- * @brief Enables registering memory on peerDevice for direct access from the current device.
  *
- * @param [in] peerDevice
+ * @param [in] peerDeviceId
  * @param [in] flags
  *
- * TODO:cudaErrorInvalidDevice error not supported in HIP, return hipErrorUnknown
- * Returns #hipSuccess, #hipErrorInvalidDevice, #hipErrorInvalidValue, #hipErrorUnknown
+ * Returns #hipSuccess, #hipErrorInvalidDevice, #hipErrorInvalidValue, 
+ * @returns #hipErrorPeerAccessAlreadyEnabled if peer access is already enabled for this device.
  */
-hipError_t  hipDeviceEnablePeerAccess ( int  peerDevice, unsigned int  flags );
+hipError_t  hipDeviceEnablePeerAccess (int  peerDeviceId, unsigned int flags);
+
+
+/**
+ * @brief Disable direct access from current device's virtual address space to memory allocations physically located on a peer device.  
+ *
+ * Returns hipErrorPeerAccessNotEnabled if direct access to memory on peerDevice has not yet been enabled from the current device.
+ *
+ * @param [in] peerDeviceId
+ *
+ * Returns #hipSuccess, #hipErrorPeerAccessNotEnabled
+ */
+hipError_t  hipDeviceDisablePeerAccess (int peerDeviceId);
+
 
 /**
  * @brief Copies memory from one device to memory on another device.
  *
  * @param [out] dst - Destination device pointer.
- * @param [in] dstDevice - Destination device
+ * @param [in] dstDeviceId - Destination device
  * @param [in] src - Source device pointer
- * @param [in] srcDevice - Source device
+ * @param [in] srcDeviceId - Source device
  * @param [in] sizeBytes - Size of memory copy in bytes
  *
  * Returns #hipSuccess, #hipErrorInvalidValue, #hipErrorInvalidDevice
  */
-hipError_t hipMemcpyPeer ( void* dst, int  dstDevice, const void* src, int  srcDevice, size_t sizeBytes );
+hipError_t hipMemcpyPeer (void* dst, int dstDeviceId, const void* src, int srcDeviceId, size_t sizeBytes);
 
 /**
  * @brief Copies memory from one device to memory on another device.
@@ -961,7 +974,7 @@ hipError_t hipMemcpyPeer ( void* dst, int  dstDevice, const void* src, int  srcD
  * Returns #hipSuccess, #hipErrorInvalidValue, #hipErrorInvalidDevice
  */
 #if __cplusplus
-hipError_t hipMemcpyPeerAsync ( void* dst, int  dstDevice, const void* src, int  srcDevice, size_t sizeBytes, hipStream_t stream=0 );
+hipError_t hipMemcpyPeerAsync ( void* dst, int  dstDeviceId, const void* src, int  srcDevice, size_t sizeBytes, hipStream_t stream=0 );
 #else
 hipError_t hipMemcpyPeerAsync(void* dst, int dstDevice, const void* src, int srcDevice, size_t sizeBytes, hipStream_t stream);
 #endif
@@ -1053,3 +1066,5 @@ hipError_t hipDriverGetVersion(int *driverVersion) ;
 /**
  *   @}
  */
+
+#endif

projects/hip/include/hcc_detail/hip_texture.h

@@ -19,7 +19,12 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
-#pragma once
+
+//#pragma once
+
+#ifndef HIP_TEXTURE_H
+#define HIP_TEXTURE_H
+
 /**
  *  @file  hcc_detail/hip_texture.h
  *  @brief HIP C++ Texture API for hcc compiler
@@ -201,3 +206,6 @@ hipError_t  hipUnbindTexture(struct texture<T, dim, readMode> *tex)
 /**
  *   @}
  */
+
+#endif
+

projects/hip/include/hcc_detail/hip_vector_types.h

@@ -25,6 +25,9 @@ THE SOFTWARE.
  *  @brief Defines the different newt vector types for HIP runtime.
  */
 
+#ifndef HIP_VECTOR_TYPES_H
+#define HIP_VECTOR_TYPES_H
+
 #if defined (__HCC__) &&  (__hcc_workweek__ < 16032)
 #error("This version of HIP requires a newer version of HCC.");
 #endif
@@ -112,7 +115,7 @@ typedef hc::short_vector::double2 double2;
 typedef hc::short_vector::double3 double3;
 typedef hc::short_vector::double4 double4;
 
-/*
+
 ///---
 // Inline functions for creating vector types from basic types
 #define ONE_COMPONENT_ACCESS(T, VT) inline VT make_ ##VT (T x) { VT t; t.x = x; return t; };
@@ -195,4 +198,7 @@ ONE_COMPONENT_ACCESS  (double, double1);
 TWO_COMPONENT_ACCESS  (double, double2);
 THREE_COMPONENT_ACCESS(double, double3);
 FOUR_COMPONENT_ACCESS (double, double4);
-*/
+
+
+#endif
+

projects/hip/include/hcc_detail/host_defines.h

@@ -25,6 +25,9 @@ THE SOFTWARE.
  *  @brief TODO-doc
  */
 
+#ifndef HOST_DEFINES_H
+#define HOST_DEFINES_H
+
 #ifdef __HCC__
 /**
  * Function and kernel markers
@@ -67,3 +70,5 @@ THE SOFTWARE.
 #define __constant__
 
 #endif
+
+#endif

projects/hip/include/hcc_detail/staging_buffer.h

@@ -17,7 +17,9 @@ OUT OF OR INN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
-#pragma once
+//#pragma once
+#ifndef STAGING_BUFFER_H
+#define STAGING_BUFFER_H
 
 #include "hsa.h"
 
@@ -58,3 +60,5 @@ private:
     hsa_signal_t     _completion_signal[_max_buffers];
     std::mutex       _copy_lock;    // provide thread-safe access 
 };
+
+#endif

projects/hip/include/hcc_detail/trace_helper.h

@@ -16,7 +16,10 @@ LIABILITY, WHETHER INN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR INN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
-#pragma once
+//#pragma once
+
+#ifndef TRACE_HELPER_H
+#define TRACE_HELPER_H
 
 #include <iostream>
 #include <iomanip>
@@ -116,3 +119,5 @@ inline std::string ToString(T first, Args... args)
 {
     return ToString(first) + ", " + ToString(args...) ;
 }
+
+#endif

projects/hip/include/hip

@@ -0,0 +1 @@
+../include

projects/hip/include/hip_runtime_api.h

@@ -138,7 +138,7 @@ typedef struct hipPointerAttribute_t {
  * @enum
  * @ingroup Enumerations
  */
-// Developer note - when updating these, update the hipErrorName and hipErrorString functions
+// Developer note - when updating these, update the hipErrorName and hipErrorString functions in NVCC and HCC paths
 typedef enum hipError_t {
      hipSuccess = 0                   ///< Successful completion.
     ,hipErrorMemoryAllocation         ///< Memory allocation error.
@@ -155,6 +155,8 @@ typedef enum hipError_t {
     ,hipErrorNoDevice                 ///< Call to hipGetDeviceCount returned 0 devices
     ,hipErrorNotReady                 ///< Indicates that asynchronous operations enqueued earlier are not ready.  This is not actually an error, but is used to distinguish from hipSuccess (which indicates completion).  APIs that return this error include hipEventQuery and hipStreamQuery.
     ,hipErrorUnknown                  ///< Unknown error.
+    ,hipErrorPeerAccessNotEnabled     ///< Peer access was never enabled from the current device.
+    ,hipErrorPeerAccessAlreadyEnabled ///< Peer access was already enabled from the current device.
     ,hipErrorRuntimeMemory            ///< HSA runtime memory call returned error.  Typically not seen in production systems.
     ,hipErrorRuntimeOther             ///< HSA runtime call other than memory returned error.  Typically not seen in production systems.
     ,hipErrorTbd                      ///< Marker that more error codes are needed.