AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'privatestaging' of https://github.com/AMDComputeLibraries/HIP-privatestaging into privatestaging

Browse Source
This commit is contained in:
Aditya Atluri
2016-03-29 11:18:09 -05:00
parent 61cd2bb399 a37a019b01
commit 39af5d9caf
17 changed files with 678 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/hcc_detail/hip_hcc.h
+213 -59
View File
@@ -47,7 +47,9 @@ extern const int release;
extern int HIP_LAUNCH_BLOCKING;
extern int HIP_PRINT_ENV;
extern int HIP_ATP_MARKER;
extern int HIP_TRACE_API;
extern int HIP_ATP;
extern int HIP_DB;
extern int HIP_STAGING_SIZE; /* size of staging buffers, in KB */
extern int HIP_STAGING_BUFFERS; // TODO - remove, two buffers should be enough.
@@ -63,8 +65,8 @@ extern int HIP_DISABLE_HW_COPY_DEP;
extern thread_local int tls_defaultDevice;
extern thread_local hipError_t tls_lastHipError;
struct ihipStream_t;
struct ihipDevice_t;
class ihipStream_t;
class ihipDevice_t;
// Color defs for debug messages:
@@ -86,6 +88,9 @@ struct ihipDevice_t;
// Stream functions will acquire a mutex before entering critical sections.
#define STREAM_THREAD_SAFE 1
#define DEVICE_THREAD_SAFE 1
// If FORCE_COPY_DEP=1 , HIP runtime will add
// synchronization for copy commands in the same stream, regardless of command type.
// If FORCE_COPY_DEP=0 data copies of the same kind (H2H, H2D, D2H, D2D) are assumed to be implicitly ordered.
@@ -111,8 +116,8 @@ struct ihipDevice_t;
// Compile code that generates trace markers for CodeXL ATP at HIP function begin/end.
// ATP is standard CodeXL format that includes timestamps for kernels, HSA RT APIs, and HIP APIs.
#ifndef COMPILE_TRACE_MARKER
#define COMPILE_TRACE_MARKER 0
#ifndef COMPILE_HIP_ATP_MARKER
#define COMPILE_HIP_ATP_MARKER 0
#endif
@@ -123,7 +128,7 @@ struct ihipDevice_t;
// Compile support for trace markers that are displayed on CodeXL GUI at start/stop of each function boundary.
// TODO - currently we print the trace message at the beginning. if we waited, we could also include return codes, and any values returned
// through ptr-to-args (ie the pointers allocated by hipMalloc).
#if COMPILE_TRACE_MARKER
#if COMPILE_HIP_ATP_MARKER
#include "AMDTActivityLogger.h"
#define SCOPED_MARKER(markerName,group,userString) amdtScopedMarker(markerName, group, userString)
#else
@@ -132,14 +137,16 @@ struct ihipDevice_t;
#endif
#if COMPILE_TRACE_MARKER || (COMPILE_HIP_TRACE_API & 0x1)
#if COMPILE_HIP_ATP_MARKER || (COMPILE_HIP_TRACE_API & 0x1)
#define API_TRACE(...)\
{\
std::string s = std::string(__func__) + " (" + ToString(__VA_ARGS__) + ')';\
if (COMPILE_HIP_DB && HIP_TRACE_API) {\
fprintf (stderr, API_COLOR "<<hip-api: %s\n" KNRM, s.c_str());\
if (HIP_ATP_MARKER || (COMPILE_HIP_DB && HIP_TRACE_API)) {\
std::string s = std::string(__func__) + " (" + ToString(__VA_ARGS__) + ')';\
if (COMPILE_HIP_DB && HIP_TRACE_API) {\
fprintf (stderr, API_COLOR "<<hip-api: %s\n" KNRM, s.c_str());\
}\
SCOPED_MARKER(s.c_str(), "HIP", NULL);\
}\
SCOPED_MARKER(s.c_str(), "HIP", NULL);\
}
#else
// Swallow API_TRACE
@@ -157,12 +164,13 @@ struct ihipDevice_t;
#define ihipLogStatus(_hip_status) \
({\
tls_lastHipError = _hip_status;\
hipError_t _local_hip_status = _hip_status; /*local copy so _hip_status only evaluated once*/ \
tls_lastHipError = _local_hip_status;\
\
if ((COMPILE_HIP_TRACE_API & 0x2) && HIP_TRACE_API) {\
fprintf(stderr, " %ship-api: %-30s ret=%2d (%s)>>\n" KNRM, (_hip_status == 0) ? API_COLOR:KRED, __func__, _hip_status, ihipErrorString(_hip_status));\
fprintf(stderr, " %ship-api: %-30s ret=%2d (%s)>>\n" KNRM, (_local_hip_status == 0) ? API_COLOR:KRED, __func__, _local_hip_status, ihipErrorString(_local_hip_status));\
}\
_hip_status;\
_local_hip_status;\
})
@@ -272,14 +280,110 @@ class FakeMutex
#if STREAM_THREAD_SAFE
typedef std::mutex StreamMutex;
#else
#warning "Stream thread-safe disabled"
typedef FakeMutex StreamMutex;
#endif
#if DEVICE_THREAD_SAFE
typedef std::mutex DeviceMutex;
#else
typedef FakeMutex DeviceMutex;
#warning "Device thread-safe disabled"
#endif
// TODO - move async copy code into stream? Stream->async-copy.
// Add PreCopy / PostCopy to manage locks?
//
//---
// Protects access to the member _data with a lock acquired on contruction/destruction.
// T must contain a _mutex field which meets the BasicLockable requirements (lock/unlock)
template<typename T>
class LockedAccessor
{
public:
LockedAccessor(T &criticalData, bool autoUnlock=true) :
_criticalData(&criticalData),
_autoUnlock(autoUnlock)
{
_criticalData->_mutex.lock();
};
~LockedAccessor()
{
if (_autoUnlock) {
_criticalData->_mutex.unlock();
}
}
void unlock()
{
_criticalData->_mutex.unlock();
}
// Syntactic sugar so -> can be used to get the underlying type.
T *operator->() { return _criticalData; };
private:
T *_criticalData;
bool _autoUnlock;
};
template <typename MUTEX_TYPE>
struct LockedBase {
// Experts-only interface for explicit locking.
// Most uses should use the lock-accessor.
void lock() { _mutex.lock(); }
void unlock() { _mutex.unlock(); }
MUTEX_TYPE _mutex;
};
template <typename MUTEX_TYPE>
class ihipStreamCriticalBase_t : public LockedBase<MUTEX_TYPE>
{
public:
ihipStreamCriticalBase_t() :
_last_command_type(ihipCommandCopyH2H),
_last_copy_signal(NULL),
_signalCursor(0),
_oldest_live_sig_id(1),
_stream_sig_id(0)
{
_signalPool.resize(HIP_STREAM_SIGNALS > 0 ? HIP_STREAM_SIGNALS : 1);
};
~ihipStreamCriticalBase_t() {
_signalPool.clear();
}
ihipStreamCriticalBase_t<StreamMutex> * mlock() { LockedBase<MUTEX_TYPE>::lock(); return this;};
public:
// Critical Data:
ihipCommand_t _last_command_type; // type of the last command
// signal of last copy command sent to the stream.
// May be NULL, indicating the previous command has completley finished and future commands don't need to create a dependency.
// Copy can be either H2D or D2H.
ihipSignal_t *_last_copy_signal;
hc::completion_future _last_kernel_future; // Completion future of last kernel command sent to GPU.
// Signal pool:
int _signalCursor;
SIGSEQNUM _oldest_live_sig_id; // oldest live seq_id, anything < this can be allocated.
std::deque<ihipSignal_t> _signalPool; // Pool of signals for use by this stream.
SIGSEQNUM _stream_sig_id; // Monotonically increasing unique signal id.
};
typedef ihipStreamCriticalBase_t<StreamMutex> ihipStreamCritical_t;
typedef LockedAccessor<ihipStreamCritical_t> LockedAccessor_StreamCrit_t;
@@ -288,69 +392,74 @@ class ihipStream_t {
public:
typedef uint64_t SeqNum_t ;
ihipStream_t(unsigned device_index, hc::accelerator_view av, SeqNum_t id, unsigned int flags);
ihipStream_t(unsigned device_index, hc::accelerator_view av, unsigned int flags);
~ihipStream_t();
// kind is hipMemcpyKind
void copySync (void* dst, const void* src, size_t sizeBytes, unsigned kind);
void copySync (LockedAccessor_StreamCrit_t &crit, void* dst, const void* src, size_t sizeBytes, unsigned kind);
void locked_copySync (void* dst, const void* src, size_t sizeBytes, unsigned kind);
void copyAsync(void* dst, const void* src, size_t sizeBytes, unsigned kind);
//---
// Thread-safe accessors - these acquire / release mutex:
bool preKernelCommand();
void postKernelCommand(hc::completion_future &kernel_future);
bool lockopen_preKernelCommand();
void lockclose_postKernelCommand(hc::completion_future &kernel_future);
int preCopyCommand(ihipSignal_t *lastCopy, hsa_signal_t *waitSignal, ihipCommand_t copyType);
int preCopyCommand(LockedAccessor_StreamCrit_t &crit, ihipSignal_t *lastCopy, hsa_signal_t *waitSignal, ihipCommand_t copyType);
void reclaimSignals_ts(SIGSEQNUM sigNum);
void wait(bool assertQueueEmpty=false);
void locked_reclaimSignals(SIGSEQNUM sigNum);
void locked_wait(bool assertQueueEmpty=false);
SIGSEQNUM locked_lastCopySeqId() {LockedAccessor_StreamCrit_t crit(_criticalData); return lastCopySeqId(crit); };
// Use this if we already have the stream critical data mutex:
void wait(LockedAccessor_StreamCrit_t &crit, bool assertQueueEmpty=false);
// Non-threadsafe accessors - must be protected by high-level stream lock:
SIGSEQNUM lastCopySeqId() { return _last_copy_signal ? _last_copy_signal->_sig_id : 0; };
ihipSignal_t * allocSignal();
// Non-threadsafe accessors - must be protected by high-level stream lock with accessor passed to function.
SIGSEQNUM lastCopySeqId (LockedAccessor_StreamCrit_t &crit) { return crit->_last_copy_signal ? crit->_last_copy_signal->_sig_id : 0; };
ihipSignal_t * allocSignal (LockedAccessor_StreamCrit_t &crit);
//-- Non-racy accessors:
// These functions access fields set at initialization time and are non-racy (so do not acquire mutex)
ihipDevice_t * getDevice() const;
StreamMutex & mutex() {return _mutex;};
ihipDevice_t * getDevice() const;
public:
//---
//Member vars - these are set at initialization:
//Public member vars - these are set at initialization and never change:
SeqNum_t _id; // monotonic sequence ID
hc::accelerator_view _av;
unsigned _flags;
private: // Critical Data. THis MUST be accessed through LockedAccessor_StreamCrit_t
ihipStreamCritical_t _criticalData;
private:
void enqueueBarrier(hsa_queue_t* queue, ihipSignal_t *depSignal);
void waitCopy(ihipSignal_t *signal);
void waitCopy(LockedAccessor_StreamCrit_t &crit, ihipSignal_t *signal);
// The unsigned return is hipMemcpyKind
unsigned resolveMemcpyDirection(bool srcInDeviceMem, bool dstInDeviceMem);
void setCopyAgents(unsigned kind, ihipCommand_t *commandType, hsa_agent_t *srcAgent, hsa_agent_t *dstAgent);
//---
unsigned _device_index; // index into the g_device array
unsigned _device_index;
ihipCommand_t _last_command_type; // type of the last command
// signal of last copy command sent to the stream.
// May be NULL, indicating the previous command has completley finished and future commands don't need to create a dependency.
// Copy can be either H2D or D2H.
ihipSignal_t *_last_copy_signal;
hc::completion_future _last_kernel_future; // Completion future of last kernel command sent to GPU.
int _signalCursor;
SIGSEQNUM _stream_sig_id; // Monotonically increasing unique signal id.
SIGSEQNUM _oldest_live_sig_id; // oldest live seq_id, anything < this can be allocated.
std::deque<ihipSignal_t> _signalPool; // Pool of signals for use by this stream.
StreamMutex _mutex;
friend std::ostream& operator<<(std::ostream& os, const ihipStream_t& s);
};
inline std::ostream& operator<<(std::ostream& os, const ihipStream_t& s)
{
os << "stream#";
os << s._device_index;
os << '.';
os << s._id;
return os;
}
//----
// Internal event structure:
@@ -379,9 +488,58 @@ struct ihipEvent_t {
//-------------------------------------------------------------------------------------------------
struct ihipDevice_t
//---
// Data that must be protected with thread-safe access
// All members are private - this class must be accessed through friend LockedAccessor which
// will lock the mutex on construction and unlock on destruction.
//
// MUTEX_TYPE is template argument so can easily convert to FakeMutex for performance or stress testing.
template <typename MUTEX_TYPE>
class ihipDeviceCriticalBase_t : LockedBase<MUTEX_TYPE>
{
public:
ihipDeviceCriticalBase_t() : _stream_id(0) {};
friend class LockedAccessor<ihipDeviceCriticalBase_t>;
std::list<ihipStream_t*> &streams() { return _streams; };
const std::list<ihipStream_t*> &const_streams() const { return _streams; };
// "Allocate" a stream ID:
ihipStream_t::SeqNum_t incStreamId() { return _stream_id++; };
private:
std::list<ihipStream_t*> _streams; // streams associated with this device.
ihipStream_t::SeqNum_t _stream_id;
};
// Note Mutex selected based on DeviceMutex
typedef ihipDeviceCriticalBase_t<DeviceMutex> ihipDeviceCritical_t;
// This type is used by functions that need access to the critical device structures.
typedef LockedAccessor<ihipDeviceCritical_t> LockedAccessor_DeviceCrit_t;
//-------------------------------------------------------------------------------------------------
// Functions which read or write the critical data are named locked_.
// ihipDevice_t does not use recursive locks so the ihip implementation must avoid calling a locked_ function from within a locked_ function.
// External functions which call several locked_ functions will acquire and release the lock for each function. if this occurs in
// performance-sensitive code we may want to refactor by adding non-locked functions and creating a new locked_ member function to call them all.
class ihipDevice_t
{
public: // Functions:
ihipDevice_t() {}; // note: calls constructor for _criticalData
void init(unsigned device_index, hc::accelerator &acc, unsigned flags);
~ihipDevice_t();
void locked_addStream(ihipStream_t *s);
void locked_removeStream(ihipStream_t *s);
void locked_reset();
void locked_waitAllStreams();
void locked_syncDefaultStream(bool waitOnSelf);
public: // Data, set at initialization:
unsigned _device_index; // index into g_devices.
hipDeviceProp_t _props; // saved device properties.
@@ -392,31 +550,27 @@ struct ihipDevice_t
// NULL has special synchronization properties with other streams.
ihipStream_t *_default_stream;
std::list<ihipStream_t*> _streams; // streams associated with this device.
unsigned _compute_units;
StagingBuffer *_staging_buffer[2]; // one buffer for each direction.
ihipStream_t::SeqNum_t _stream_id;
unsigned _device_flags;
public:
void init(unsigned device_index, hc::accelerator acc, unsigned flags);
~ihipDevice_t();
void reset();
private:
hipError_t getProperties(hipDeviceProp_t* prop);
void waitAllStreams();
void syncDefaultStream(bool waitOnSelf);
private:
private: // Critical data, protected with locked access:
// Members of _protected data MUST be accessed through the LockedAccessor.
// Search for LockedAccessor<ihipDeviceCritical_t> for examples; do not access _criticalData directly.
ihipDeviceCritical_t _criticalData;
};
// Global initialization.
// Global variable definition:
extern std::once_flag hip_initialized;
extern ihipDevice_t *g_devices; // Array of all non-emulated (ie GPU) accelerators in the system.
extern bool g_visible_device; // Set the flag when HIP_VISIBLE_DEVICES is set
include/hcc_detail/hip_runtime_api.h
+3 -9
View File
@@ -449,10 +449,7 @@ hipError_t hipStreamCreateWithFlags(hipStream_t *stream, unsigned int flags);
* @see hipStreamDestroy
*
*/
static inline hipError_t hipStreamCreate(hipStream_t *stream)
{
return hipStreamCreateWithFlags(stream, hipStreamDefault);
}
hipError_t hipStreamCreate(hipStream_t *stream);
/**
@@ -550,13 +547,10 @@ hipError_t hipEventCreateWithFlags(hipEvent_t* event, unsigned flags);
/**
* Create an event
*
* @param[in] event Creates an event
* @param[in,out] event Returns the newly created event.
*
*/
static inline hipError_t hipEventCreate(hipEvent_t* event)
{
return hipEventCreateWithFlags(event, 0);
}
hipError_t hipEventCreate(hipEvent_t* event);
/**
include/hcc_detail/trace_helper.h
+37 -7
View File
@@ -44,18 +44,45 @@ inline std::string ToHexString(T v)
//---
// Template overloads for ToString to handle various types:
// Note these use C++11 variadic templates
// Template overloads for ToString to handle specific types
// This is the default which works for most types:
template <typename T>
inline std::string ToString(T v) {
inline std::string ToString(T v)
{
std::ostringstream ss;
ss << v;
return ss.str();
};
// hipEvent_t specialization. TODO - maybe add an event ID for debug?
template <>
inline std::string ToString(hipMemcpyKind v) {
inline std::string ToString(hipEvent_t v)
{
return ToString(&v);
};
// hipStream_t
template <>
inline std::string ToString(hipStream_t v)
{
std::ostringstream ss;
if (v == NULL) {
ss << "stream:<null>";
} else {
ss << *v;
}
return ss.str();
};
// hipMemcpyKind specialization
template <>
inline std::string ToString(hipMemcpyKind v)
{
switch(v) {
CASE_STR(hipMemcpyHostToHost);
CASE_STR(hipMemcpyHostToDevice);
@@ -68,13 +95,15 @@ inline std::string ToString(hipMemcpyKind v) {
template <>
inline std::string ToString(hipError_t v) {
inline std::string ToString(hipError_t v)
{
return ihipErrorString(v);
};
// Catch empty arguments case
inline std::string ToString() {
inline std::string ToString()
{
return ("");
}
@@ -83,6 +112,7 @@ inline std::string ToString() {
// C++11 variadic template - peels off first argument, converts to string, and calls itself again to peel the next arg.
// Strings are automatically separated by comma+space.
template <typename T, typename... Args>
inline std::string ToString(T first, Args... args) {
inline std::string ToString(T first, Args... args)
{
return ToString(first) + ", " + ToString(args...) ;
}
include/hip_runtime_api.h
+1 -1
View File
@@ -222,7 +222,7 @@ static inline hipError_t hipMalloc ( T** devPtr, size_t size)
// Provide an override to automatically typecast the pointer type from void**, and also provide a default for the flags.
template<class T>
static inline hipError_t hipHostMalloc( T** ptr, size_t size, unsigned int flags = 0)
static inline hipError_t hipHostMalloc( T** ptr, size_t size, unsigned int flags = hipHostMallocDefault)
{
return hipHostMalloc((void**)ptr, size, flags);
}