AI/rocm-systems
2
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'amd-master' into amd-master-next

Browse Source 
Change-Id: I3094c15008093f2072bcd38aca4ea90aeae2d97b


[ROCm/hip commit: 2af31479e2]
This commit is contained in:
Maneesh Gupta committed 2020-04-09 06:31:00 -04:00
1 parent c34599e77c
commit 4e1f6f2a0e
67 files changed
+2278 -601
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

projects/hip/src/code_object_bundle.cpp
-34
View File
@@ -1,34 +0,0 @@
#include "../include/hip/hcc_detail/code_object_bundle.hpp"
#include <hsa/hsa.h>
#include <algorithm>
#include <cstddef>
#include <cstdint>
#include <string>
#include <vector>
using namespace std;
// CREATORS
hip_impl::Bundled_code_header::Bundled_code_header(const vector<char>& x)
: Bundled_code_header{x.cbegin(), x.cend()} {}
hip_impl::Bundled_code_header::Bundled_code_header(
const void* p) { // This is a pretty terrible interface, useful only because
// hipLoadModuleData is so poorly specified (for no fault of its own).
if (!p) return;
if (!valid(*static_cast<const Bundled_code_header*>(p))) return;
auto ph = static_cast<const Header_*>(p);
size_t sz = sizeof(Header_) + ph->bundle_cnt_ * sizeof(Bundled_code::Header);
auto pb = static_cast<const char*>(p) + sizeof(Header_);
auto n = ph->bundle_cnt_;
while (n--) {
sz += reinterpret_cast<const Bundled_code::Header*>(pb)->bundle_sz;
pb += sizeof(Bundled_code::Header);
}
read(static_cast<const char*>(p), static_cast<const char*>(p) + sz, *this);
}
projects/hip/src/code_object_bundle.inl
+203
View File
@@ -0,0 +1,203 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#pragma once
#include <hsa/hsa.h>
#include <algorithm>
#include <cstdint>
#include <istream>
#include <iterator>
#include <string>
#include <utility>
#include <vector>
#include <unordered_set>
namespace hip_impl {
#if !defined(DISABLE_REDUCED_GPU_BLOB_COPY)
std::unordered_set<std::string>& get_all_gpuarch();
#endif
inline
std::string transmogrify_triple(const std::string& triple)
{
static constexpr const char old_prefix[]{"hcc-amdgcn--amdhsa-gfx"};
static constexpr const char new_prefix[]{"hcc-amdgcn-amd-amdhsa--gfx"};
if (triple.find(old_prefix) == 0) {
return new_prefix + triple.substr(sizeof(old_prefix) - 1);
}
return (triple.find(new_prefix) == 0) ? triple : "";
}
inline
std::string isa_name(std::string triple)
{
static constexpr const char offload_prefix[]{"hcc-"};
triple = transmogrify_triple(triple);
if (triple.empty()) return {};
triple.erase(0, sizeof(offload_prefix) - 1);
return triple;
}
inline
hsa_isa_t triple_to_hsa_isa(const std::string& triple) {
const std::string isa{isa_name(std::move(triple))};
if (isa.empty()) return hsa_isa_t({});
hsa_isa_t r{};
if(HSA_STATUS_SUCCESS != hsa_isa_from_name(isa.c_str(), &r)) {
r.handle = 0;
}
return r;
}
struct Bundled_code {
union Header {
struct {
std::uint64_t offset;
std::uint64_t bundle_sz;
std::uint64_t triple_sz;
};
char cbuf[sizeof(offset) + sizeof(bundle_sz) + sizeof(triple_sz)];
} header;
std::string triple;
std::string blob;
};
#define magic_string_ "__CLANG_OFFLOAD_BUNDLE__"
class Bundled_code_header {
// DATA - STATICS
static constexpr auto magic_string_sz_ = sizeof(magic_string_) - 1;
// DATA
union Header_ {
struct {
char bundler_magic_string_[magic_string_sz_];
std::uint64_t bundle_cnt_;
};
char cbuf_[sizeof(bundler_magic_string_) + sizeof(bundle_cnt_)];
} header_;
std::vector<Bundled_code> bundles_;
// FRIENDS - MANIPULATORS
template <typename RandomAccessIterator>
friend inline bool read(RandomAccessIterator f, RandomAccessIterator l,
Bundled_code_header& x) {
if (f == l) return false;
std::copy_n(f, sizeof(x.header_.cbuf_), x.header_.cbuf_);
if (valid(x)) {
x.bundles_.resize(x.header_.bundle_cnt_);
auto it = f + sizeof(x.header_.cbuf_);
for (auto&& y : x.bundles_) {
std::copy_n(it, sizeof(y.header.cbuf), y.header.cbuf);
it += sizeof(y.header.cbuf);
y.triple.assign(it, it + y.header.triple_sz);
#ifdef DISABLE_REDUCED_GPU_BLOB_COPY
std::copy_n(f + y.header.offset, y.header.bundle_sz, std::back_inserter(y.blob));
#else
auto& gpuArch = get_all_gpuarch();
auto itgpuArch = std::find(gpuArch.begin(),gpuArch.end(),y.triple);
if (itgpuArch != gpuArch.end()){
std::copy_n(f + y.header.offset, y.header.bundle_sz, std::back_inserter(y.blob));
}
#endif
it += y.header.triple_sz;
x.bundled_code_size = std::max(x.bundled_code_size,
y.header.offset + y.header.bundle_sz);
}
return true;
}
return false;
}
friend inline bool read(const std::vector<char>& blob, Bundled_code_header& x) {
return read(blob.cbegin(), blob.cend(), x);
}
friend inline bool read(std::istream& is, Bundled_code_header& x) {
return read(
std::vector<char>{std::istreambuf_iterator<char>{is}, std::istreambuf_iterator<char>{}},
x);
}
// FRIENDS - ACCESSORS
friend inline bool valid(const Bundled_code_header& x) {
const std::string ms = {magic_string_};
return std::equal(ms.begin(), ms.end(), x.header_.bundler_magic_string_);
}
friend inline const std::vector<Bundled_code>& bundles(const Bundled_code_header& x) {
return x.bundles_;
}
public:
// CREATORS
Bundled_code_header() = default;
template <typename RandomAccessIterator>
Bundled_code_header(RandomAccessIterator f, RandomAccessIterator l);
explicit Bundled_code_header(const std::vector<char>& blob)
: Bundled_code_header{blob.cbegin(), blob.cend()} {}
explicit Bundled_code_header(const void* maybe_blob) {
// This is a pretty terrible interface, useful only because
// hipLoadModuleData is so poorly specified (for no fault of its own).
if (!maybe_blob) return;
if (!valid(*static_cast<const Bundled_code_header*>(maybe_blob))) return;
auto ph = static_cast<const Header_*>(maybe_blob);
size_t sz = sizeof(Header_) + ph->bundle_cnt_ * sizeof(Bundled_code::Header);
auto pb = static_cast<const char*>(maybe_blob) + sizeof(Header_);
auto n = ph->bundle_cnt_;
while (n--) {
sz += reinterpret_cast<const Bundled_code::Header*>(pb)->bundle_sz;
pb += sizeof(Bundled_code::Header);
}
read(static_cast<const char*>(maybe_blob), static_cast<const char*>(maybe_blob) + sz, *this);
}
Bundled_code_header(const Bundled_code_header&) = default;
Bundled_code_header(Bundled_code_header&&) = default;
~Bundled_code_header() = default;
// MANIPULATORS
Bundled_code_header& operator=(const Bundled_code_header&) = default;
Bundled_code_header& operator=(Bundled_code_header&&) = default;
size_t bundled_code_size = 0;
};
// CREATORS
template <typename RandomAccessIterator>
Bundled_code_header::Bundled_code_header(RandomAccessIterator f, RandomAccessIterator l)
: Bundled_code_header{} {
read(f, l, *this);
}
} // Namespace hip_impl.
projects/hip/src/hip_clang.cpp
+2 -2
View File
@@ -51,7 +51,7 @@ __hipRegisterFatBinary(const void* data)
return nullptr;
}
auto modules = new std::vector<hipModule_t>{g_deviceCnt};
auto modules = new std::vector<hipModule_t>(g_deviceCnt);
if (!modules) {
return nullptr;
}
@@ -136,7 +136,7 @@ extern "C" void __hipRegisterFunction(
int* wSize)
{
HIP_INIT_API(NONE, modules, hostFunction, deviceFunction, deviceName);
std::vector<hipFunction_t> functions{g_deviceCnt};
std::vector<hipFunction_t> functions(g_deviceCnt);
assert(modules && modules->size() >= g_deviceCnt);
for (int deviceId = 0; deviceId < g_deviceCnt; ++deviceId) {
projects/hip/src/hip_device.cpp
+12
View File
@@ -310,6 +310,18 @@ hipError_t ihipDeviceGetAttribute(int* pi, hipDeviceAttribute_t attr, int device
case hipDeviceAttributeCooperativeMultiDeviceLaunch:
*pi = prop->cooperativeMultiDeviceLaunch;
break;
case hipDeviceAttributeCooperativeMultiDeviceUnmatchedFunc:
*pi = prop->cooperativeMultiDeviceUnmatchedFunc;
break;
case hipDeviceAttributeCooperativeMultiDeviceUnmatchedGridDim:
*pi = prop->cooperativeMultiDeviceUnmatchedGridDim;
break;
case hipDeviceAttributeCooperativeMultiDeviceUnmatchedBlockDim:
*pi = prop->cooperativeMultiDeviceUnmatchedBlockDim;
break;
case hipDeviceAttributeCooperativeMultiDeviceUnmatchedSharedMem:
*pi = prop->cooperativeMultiDeviceUnmatchedSharedMem;
break;
case hipDeviceAttributeMaxPitch:
*pi = prop->memPitch;
break;
projects/hip/src/hip_hcc.cpp
+16 -23
View File
@@ -677,7 +677,7 @@ hsa_status_t get_pool_info(hsa_amd_memory_pool_t pool, void* data) {
break;
case HSA_REGION_SEGMENT_GROUP:
err = hsa_amd_memory_pool_get_info(pool, HSA_AMD_MEMORY_POOL_INFO_SIZE,
&(p_prop->sharedMemPerBlock));
&(p_prop->maxSharedMemoryPerMultiProcessor));
break;
default:
break;
@@ -835,10 +835,8 @@ hipError_t ihipDevice_t::initProperties(hipDeviceProp_t* prop) {
hsa_region_t* am_region = static_cast<hsa_region_t*>(_acc.get_hsa_am_region());
err = hsa_region_get_info(*am_region, HSA_REGION_INFO_SIZE, &prop->totalGlobalMem);
DeviceErrorCheck(err);
// maxSharedMemoryPerMultiProcessor should be as the same as group memory size.
// Group memory will not be paged out, so, the physical memory size is the total shared memory
// size, and also equal to the group pool size.
prop->maxSharedMemoryPerMultiProcessor = prop->totalGlobalMem;
// Current GPUs allow a workgroup to use all of LDS in a CU, so these two are equal.
prop->sharedMemPerBlock = prop->maxSharedMemoryPerMultiProcessor;
// Get Max memory clock frequency
err =
@@ -897,9 +895,16 @@ hipError_t ihipDevice_t::initProperties(hipDeviceProp_t* prop) {
prop->integrated = 1;
}
// Enable the cooperative group for gfx9+
prop->cooperativeLaunch = (prop->gcnArch < 900) ? 0 : 1;
prop->cooperativeMultiDeviceLaunch = (prop->gcnArch < 900) ? 0 : 1;
// Enable the cooperative group for GPUs that support all the required features
err = hsa_agent_get_info(_hsaAgent, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_COOPERATIVE_QUEUES,
&prop->cooperativeLaunch);
DeviceErrorCheck(err);
prop->cooperativeMultiDeviceLaunch = prop->cooperativeLaunch;
prop->cooperativeMultiDeviceUnmatchedFunc = prop->cooperativeMultiDeviceLaunch;
prop->cooperativeMultiDeviceUnmatchedGridDim = prop->cooperativeMultiDeviceLaunch;
prop->cooperativeMultiDeviceUnmatchedBlockDim = prop->cooperativeMultiDeviceLaunch;
prop->cooperativeMultiDeviceUnmatchedSharedMem = prop->cooperativeMultiDeviceLaunch;
err = hsa_agent_get_info(_hsaAgent, (hsa_agent_info_t)HSA_EXT_AGENT_INFO_IMAGE_1D_MAX_ELEMENTS,
&prop->maxTexture1D);
@@ -1515,20 +1520,6 @@ hipError_t ihipStreamSynchronize(TlsData *tls, hipStream_t stream) {
return e;
}
void ihipStreamCallbackHandler(ihipStreamCallback_t* cb) {
hipError_t e = hipSuccess;
// Synchronize stream
tprintf(DB_SYNC, "ihipStreamCallbackHandler wait on stream %s\n",
ToString(cb->_stream).c_str());
GET_TLS();
e = ihipStreamSynchronize(tls, cb->_stream);
// Call registered callback function
cb->_callback(cb->_stream, e, cb->_userData);
delete cb;
}
//---
// Get the stream to use for a command submission.
//
@@ -1619,7 +1610,9 @@ void ihipPrintKernelLaunch(const char* kernelName, const grid_launch_parm* lp,
// Allows runtime to track some information about the stream.
hipStream_t ihipPreLaunchKernel(hipStream_t stream, dim3 grid, dim3 block, grid_launch_parm* lp,
const char* kernelNameStr, bool lockAcquired) {
stream = ihipSyncAndResolveStream(stream, lockAcquired);
if (stream == nullptr || stream != stream->getCtx()->_defaultStream) {
stream = ihipSyncAndResolveStream(stream, lockAcquired);
}
lp->grid_dim.x = grid.x;
lp->grid_dim.y = grid.y;
lp->grid_dim.z = grid.z;
projects/hip/src/hip_hcc_internal.h
-14
View File
@@ -654,19 +654,6 @@ class ihipStream_t {
};
//----
// Internal structure for stream callback handler
class ihipStreamCallback_t {
public:
ihipStreamCallback_t(hipStream_t stream, hipStreamCallback_t callback, void* userData)
: _stream(stream), _callback(callback), _userData(userData) {
};
hipStream_t _stream;
hipStreamCallback_t _callback;
void* _userData;
};
//----
// Internal event structure:
enum hipEventStatus_t {
@@ -980,7 +967,6 @@ hipError_t hipModuleGetFunctionEx(hipFunction_t* hfunc, hipModule_t hmod,
hipStream_t ihipSyncAndResolveStream(hipStream_t, bool lockAcquired = 0);
hipError_t ihipStreamSynchronize(TlsData *tls, hipStream_t stream);
void ihipStreamCallbackHandler(ihipStreamCallback_t* cb);
/**
* @brief Copies the memory address and size of symbol @p symbolName
projects/hip/src/hip_memory.cpp
+68 -33
View File
@@ -309,31 +309,52 @@ void generic_copy(void* __restrict dst, const void* __restrict src, size_t n,
if (di.size == is_cpu_owned) return d2h_copy(dst, src, n, si);
if (si.size == is_cpu_owned) return h2d_copy(dst, src, n, di);
throwing_result_check(hsa_amd_agents_allow_access(1u, &si.agentOwner,
nullptr,
di.agentBaseAddress),
__FILE__, __func__, __LINE__);
return do_copy(dst, src, n, di.agentOwner, si.agentOwner);
hsa_status_t res = hsa_amd_agents_allow_access(1u, &si.agentOwner,
nullptr, di.agentBaseAddress);
if (res == HSA_STATUS_SUCCESS){
return do_copy(dst, src, n, di.agentOwner, si.agentOwner);
}
// If devices do not have access then fallback mechanism will be used
// copy will be slower
throwing_result_check(hsa_memory_copy(dst,src,n), __FILE__, __func__, __LINE__);
}
inline
void memcpy_impl(void* __restrict dst, const void* __restrict src, size_t n,
hipMemcpyKind k) {
auto si{info(src)};
auto di{info(dst)};
if (!is_large_BAR){
// Pointer info takes presidence over hipMemcpyKind
// if there is mismatch b/w Memcpy kind and dst/src pointer
// E.g. dst(host pointer),src(device pointer) and hipMemcpyKind set as hipMemcpyHostToDevice
if (di.size == is_cpu_owned && si.size == is_cpu_owned)
k = hipMemcpyHostToHost;
else if (si.size == is_cpu_owned && di.size != is_cpu_owned)
k = hipMemcpyHostToDevice;
else if (di.size == is_cpu_owned && si.size != is_cpu_owned)
k = hipMemcpyDeviceToHost;
else
k = hipMemcpyDeviceToDevice;
}
switch (k) {
case hipMemcpyHostToHost: std::memcpy(dst, src, n); break;
case hipMemcpyHostToDevice: return h2d_copy(dst, src, n, info(dst));
case hipMemcpyDeviceToHost: return d2h_copy(dst, src, n, info(src));
case hipMemcpyHostToDevice: return h2d_copy(dst, src, n, di);
case hipMemcpyDeviceToHost: return d2h_copy(dst, src, n, si);
case hipMemcpyDeviceToDevice: {
const auto di{info(dst)};
const auto si{info(src)};
throwing_result_check(hsa_amd_agents_allow_access(1u, &si.agentOwner,
nullptr,
di.agentBaseAddress),
__FILE__, __func__, __LINE__);
return do_copy(dst, src, n, di.agentOwner, si.agentOwner);
hsa_status_t res = hsa_amd_agents_allow_access(1u, &si.agentOwner,
nullptr, di.agentBaseAddress);
if (res == HSA_STATUS_SUCCESS){
return do_copy(dst, src, n, di.agentOwner, si.agentOwner);
}
// If devices do not have access then fallback mechanism will be used
// copy will be slower
throwing_result_check(hsa_memory_copy(dst,src,n), __FILE__, __func__, __LINE__);
break;
}
default: return generic_copy(dst, src, n, info(dst), info(src));
default: return generic_copy(dst, src, n, di, si);
}
}
@@ -478,6 +499,10 @@ void* allocAndSharePtr(const char* msg, size_t sizeBytes, ihipCtx_t* ctx, bool s
hipError_t ihipHostMalloc(TlsData *tls, void** ptr, size_t sizeBytes, unsigned int flags) {
hipError_t hip_status = hipSuccess;
if (sizeBytes == 0) {
return hipSuccess;
}
if (HIP_SYNC_HOST_ALLOC) {
hipDeviceSynchronize();
}
@@ -485,10 +510,6 @@ hipError_t ihipHostMalloc(TlsData *tls, void** ptr, size_t sizeBytes, unsigned i
auto ctx = ihipGetTlsDefaultCtx();
if ((ctx == nullptr) || (ptr == nullptr)) {
hip_status = hipErrorInvalidValue;
}
else if (sizeBytes == 0) {
hip_status = hipSuccess;
// TODO - should size of 0 return err or be siliently ignored?
} else {
unsigned trueFlags = flags;
if (flags == hipHostMallocDefault) {
@@ -673,14 +694,15 @@ hipError_t hipMalloc(void** ptr, size_t sizeBytes) {
HIP_SET_DEVICE();
hipError_t hip_status = hipSuccess;
if (sizeBytes == 0) {
if (ptr) *ptr = NULL;
return ihipLogStatus(hipSuccess);
}
auto ctx = ihipGetTlsDefaultCtx();
// return NULL pointer when malloc size is 0
if ( nullptr == ctx || nullptr == ptr) {
hip_status = hipErrorInvalidValue;
}
else if (sizeBytes == 0) {
*ptr = NULL;
hip_status = hipSuccess;
} else {
auto device = ctx->getWriteableDevice();
*ptr = hip_internal::allocAndSharePtr("device_mem", sizeBytes, ctx, false /*shareWithAll*/,
@@ -700,14 +722,15 @@ hipError_t hipExtMallocWithFlags(void** ptr, size_t sizeBytes, unsigned int flag
HIP_SET_DEVICE();
#if (__hcc_workweek__ >= 19115)
if (sizeBytes == 0) {
if (ptr) *ptr = NULL;
return ihipLogStatus(hipSuccess);
}
hipError_t hip_status = hipSuccess;
auto ctx = ihipGetTlsDefaultCtx();
// return NULL pointer when malloc size is 0
if (sizeBytes == 0) {
*ptr = NULL;
hip_status = hipSuccess;
} else if ((ctx == nullptr) || (ptr == nullptr)) {
if ((ctx == nullptr) || (ptr == nullptr)) {
hip_status = hipErrorInvalidValue;
} else {
unsigned amFlags = 0;
@@ -736,6 +759,9 @@ hipError_t hipExtMallocWithFlags(void** ptr, size_t sizeBytes, unsigned int flag
hipError_t hipHostMalloc(void** ptr, size_t sizeBytes, unsigned int flags) {
HIP_INIT_SPECIAL_API(hipHostMalloc, (TRACE_MEM), ptr, sizeBytes, flags);
HIP_SET_DEVICE();
if (sizeBytes == 0) {
return ihipLogStatus(hipSuccess);
}
hipError_t hip_status = hipSuccess;
hip_status = hip_internal::ihipHostMalloc(tls, ptr, sizeBytes, flags);
return ihipLogStatus(hip_status);
@@ -744,6 +770,9 @@ hipError_t hipHostMalloc(void** ptr, size_t sizeBytes, unsigned int flags) {
hipError_t hipMallocManaged(void** devPtr, size_t size, unsigned int flags) {
HIP_INIT_SPECIAL_API(hipMallocManaged, (TRACE_MEM), devPtr, size, flags);
HIP_SET_DEVICE();
if (size == 0) {
return ihipLogStatus(hipSuccess);
}
hipError_t hip_status = hipSuccess;
if(flags != hipMemAttachGlobal)
hip_status = hipErrorInvalidValue;
@@ -1224,6 +1253,7 @@ hipError_t hipMemcpyToSymbol(void* dst, const void* src, size_t count,
tprintf(DB_MEM, " symbol '%s' resolved to address:%p\n", symbol_name, dst);
if (count == 0) return ihipLogStatus(hipSuccess);
if (dst == nullptr) {
return ihipLogStatus(hipErrorInvalidSymbol);
}
@@ -1246,6 +1276,7 @@ hipError_t hipMemcpyFromSymbol(void* dst, const void* src, size_t count,
tprintf(DB_MEM, " symbol '%s' resolved to address:%p\n", symbol_name, dst);
if (count == 0) return ihipLogStatus(hipSuccess);
if (src == nullptr || dst == nullptr) {
return ihipLogStatus(hipErrorInvalidSymbol);
}
@@ -1269,6 +1300,7 @@ hipError_t hipMemcpyToSymbolAsync(void* dst, const void* src, size_t count,
tprintf(DB_MEM, " symbol '%s' resolved to address:%p\n", symbol_name, dst);
if (count == 0) return ihipLogStatus(hipSuccess);
if (dst == nullptr) {
return ihipLogStatus(hipErrorInvalidSymbol);
}
@@ -1301,6 +1333,7 @@ hipError_t hipMemcpyFromSymbolAsync(void* dst, const void* src, size_t count,
tprintf(DB_MEM, " symbol '%s' resolved to address:%p\n", symbol_name, src);
if (count == 0) return ihipLogStatus(hipSuccess);
if (src == nullptr || dst == nullptr) {
return ihipLogStatus(hipErrorInvalidSymbol);
}
@@ -1592,6 +1625,7 @@ hipError_t ihipMemcpy3D(const struct hipMemcpy3DParms* p, hipStream_t stream, bo
srcXoffset = p->srcPos.x;
srcYoffset = p->srcPos.y;
srcZoffset = p->srcPos.z;
if (copyWidth == 0) return hipSuccess;
if (p->dstArray != nullptr) {
if ((p->dstArray->isDrv == true) ||( p->dstPtr.ptr!= nullptr)){
return hipErrorInvalidValue;
@@ -1933,6 +1967,7 @@ hipError_t getLockedPointer(void *hostPtr, size_t dataLen, void **devicePtrPtr)
// TODO - review and optimize
hipError_t ihipMemcpy2D(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width,
size_t height, hipMemcpyKind kind) {
if (height == 0 || width == 0) return hipSuccess;
if (dst == nullptr || src == nullptr || width > dpitch || width > spitch) return hipErrorInvalidValue;
hipStream_t stream = ihipSyncAndResolveStream(hipStreamNull);
@@ -1989,6 +2024,7 @@ hipError_t hipMemcpy2D(void* dst, size_t dpitch, const void* src, size_t spitch,
hipError_t ihipMemcpy2DAsync(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width,
size_t height, hipMemcpyKind kind, hipStream_t stream) {
if (height == 0 || width == 0) return hipSuccess;
if (dst == nullptr || src == nullptr || width > dpitch || width > spitch) return hipErrorInvalidValue;
hipError_t e = hipSuccess;
int isLockedOrD2D = 0;
@@ -2043,6 +2079,7 @@ hipError_t ihip2dOffsetMemcpy(void* dst, size_t dpitch, const void* src, size_t
size_t height, size_t srcXOffsetInBytes, size_t srcYOffset,
size_t dstXOffsetInBytes, size_t dstYOffset,hipMemcpyKind kind,
hipStream_t stream, bool isAsync) {
if (height == 0 || width == 0) return hipSuccess;
if((spitch < width + srcXOffsetInBytes) || (srcYOffset >= height)){
return hipErrorInvalidValue;
} else if((dpitch < width + dstXOffsetInBytes) || (dstYOffset >= height)){
@@ -2061,6 +2098,7 @@ hipError_t ihipMemcpyParam2D(const hip_Memcpy2D* pCopy, hipStream_t stream, bool
if (pCopy == nullptr) {
return hipErrorInvalidValue;
}
if (pCopy->Height == 0 || pCopy->WidthInBytes == 0) return hipSuccess;
void* dst; const void* src;
size_t spitch = pCopy->srcPitch;
size_t dpitch = pCopy->dstPitch;
@@ -2140,6 +2178,7 @@ hipError_t hipMemcpy2DFromArray( void* dst, size_t dpitch, hipArray_const_t src,
hipError_t hipMemcpy2DFromArrayAsync( void* dst, size_t dpitch, hipArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, hipMemcpyKind kind, hipStream_t stream ){
HIP_INIT_SPECIAL_API(hipMemcpy2DFromArrayAsync, (TRACE_MCMD), dst, dpitch, src, wOffset, hOffset, width, height, kind, stream);
size_t byteSize;
if (height == 0 || width == 0) return ihipLogStatus(hipSuccess);
if(src) {
switch (src->desc.f) {
case hipChannelFormatKindSigned:
@@ -2239,8 +2278,6 @@ hipError_t hipMemGetInfo(size_t* free, size_t* total) {
auto device = ctx->getWriteableDevice();
if (total) {
*total = device->_props.totalGlobalMem;
} else {
e = hipErrorInvalidValue;
}
if (free) {
@@ -2263,8 +2300,6 @@ hipError_t hipMemGetInfo(size_t* free, size_t* total) {
} else {
return ihipLogStatus(hipErrorInvalidValue);
}
} else {
e = hipErrorInvalidValue;
}
} else {
projects/hip/src/hip_module.cpp
+316 -159
View File
@@ -50,7 +50,7 @@ THE SOFTWARE.
#include <unordered_map>
#include <utility>
#include <vector>
#include "../include/hip/hcc_detail/code_object_bundle.hpp"
#include "code_object_bundle.inl"
#include "hip_fatbin.h"
// TODO Use Pool APIs from HCC to get memory regions.
@@ -140,7 +140,7 @@ hipError_t ihipModuleLaunchKernel(TlsData *tls, hipFunction_t f, uint32_t global
uint32_t localWorkSizeZ, size_t sharedMemBytes,
hipStream_t hStream, void** kernelParams, void** extra,
hipEvent_t startEvent, hipEvent_t stopEvent, uint32_t flags, bool isStreamLocked = 0,
void** impCoopParams = 0) {
void** impCoopParams = 0, hc::accelerator_view* coopAV = 0) {
using namespace hip_impl;
auto ctx = ihipGetTlsDefaultCtx();
@@ -192,8 +192,8 @@ hipError_t ihipModuleLaunchKernel(TlsData *tls, hipFunction_t f, uint32_t global
if (impCoopParams) {
const auto p{static_cast<const char*>(*impCoopParams)};
// The sixth index is for multi-grid synchronization
kernargs.insert((kernargs.cend() - padSize - HIP_IMPLICIT_KERNARG_SIZE) + 6 * HIP_IMPLICIT_KERNARG_ALIGNMENT,
p, p + HIP_IMPLICIT_KERNARG_ALIGNMENT);
copy(p, p + HIP_IMPLICIT_KERNARG_ALIGNMENT,
(kernargs.end() - HIP_IMPLICIT_KERNARG_SIZE) + 6 * HIP_IMPLICIT_KERNARG_ALIGNMENT);
}
/*
@@ -245,6 +245,10 @@ hipError_t ihipModuleLaunchKernel(TlsData *tls, hipFunction_t f, uint32_t global
hc::completion_future cf;
if (coopAV) {
lp.av = coopAV;
}
lp.av->dispatch_hsa_kernel(&aql, kernargs.data(), kernargs.size(),
(startEvent || stopEvent) ? &cf : nullptr
#if (__hcc_workweek__ > 17312)
@@ -326,22 +330,18 @@ hipError_t ihipExtLaunchMultiKernelMultiDevice(hipLaunchParams* launchParamsList
return hipErrorInvalidValue;
}
hipFunction_t* kds = reinterpret_cast<hipFunction_t*>(malloc(sizeof(hipFunction_t) * numDevices));
if (kds == nullptr) {
return hipErrorNotInitialized;
}
std::vector<hipFunction_t> kds(numDevices,0);
// prepare all kernel descriptors for each device as all streams will be locked in the next loop
for (int i = 0; i < numDevices; ++i) {
const hipLaunchParams& lp = launchParamsList[i];
if (lp.stream == nullptr) {
free(kds);
return hipErrorNotInitialized;
}
kds[i] = ps.kernel_descriptor(reinterpret_cast<std::uintptr_t>(lp.func),
hip_impl::target_agent(lp.stream));
if (kds[i] == nullptr) {
free(kds);
return hipErrorInvalidValue;
}
if (!kds[i]->_kernarg_layout.empty()) continue;
@@ -396,8 +396,6 @@ hipError_t ihipExtLaunchMultiKernelMultiDevice(hipLaunchParams* launchParamsList
#endif
}
free(kds);
return result;
}
@@ -409,6 +407,90 @@ hipError_t hipExtLaunchMultiKernelMultiDevice(hipLaunchParams* launchParamsList,
return ihipExtLaunchMultiKernelMultiDevice(launchParamsList, numDevices, flags, ps);
}
void getGprsLdsUsage(hipFunction_t f, size_t* usedVGPRS, size_t* usedSGPRS, size_t* usedLDS)
{
if (f->_is_code_object_v3) {
const auto header = reinterpret_cast<const amd_kernel_code_v3_t*>(f->_header);
// GRANULATED_WAVEFRONT_VGPR_COUNT is specified in 0:5 bits of COMPUTE_PGM_RSRC1
// the granularity for gfx6-gfx9 is max(0, ceil(vgprs_used / 4) - 1)
*usedVGPRS = ((header->compute_pgm_rsrc1 & 0x3F) + 1) << 2;
// GRANULATED_WAVEFRONT_SGPR_COUNT is specified in 6:9 bits of COMPUTE_PGM_RSRC1
// the granularity for gfx9+ is 2 * max(0, ceil(sgprs_used / 16) - 1)
*usedSGPRS = ((((header->compute_pgm_rsrc1 & 0x3C0) >> 6) >> 1) + 1) << 4;
*usedLDS = header->group_segment_fixed_size;
}
else {
const auto header = f->_header;
// VGPRs granularity is 4
*usedVGPRS = ((header->workitem_vgpr_count + 3) >> 2) << 2;
// adding 2 to take into account the 2 VCC registers & handle the granularity of 16
*usedSGPRS = header->wavefront_sgpr_count + 2;
*usedSGPRS = ((*usedSGPRS + 15) >> 4) << 4;
*usedLDS = header->workgroup_group_segment_byte_size;
}
}
static hipError_t ihipOccupancyMaxActiveBlocksPerMultiprocessor(
TlsData *tls, uint32_t* numBlocks, hipFunction_t f, uint32_t blockSize, size_t dynSharedMemPerBlk)
{
using namespace hip_impl;
auto ctx = ihipGetTlsDefaultCtx();
if (ctx == nullptr) {
return hipErrorInvalidDevice;
}
if (numBlocks == nullptr) {
return hipErrorInvalidValue;
}
hipDeviceProp_t prop{};
ihipGetDeviceProperties(&prop, ihipGetTlsDefaultCtx()->getDevice()->_deviceId);
if (blockSize > prop.maxThreadsPerBlock) {
*numBlocks = 0;
return hipSuccess;
}
prop.regsPerBlock = prop.regsPerBlock ? prop.regsPerBlock : 64 * 1024;
size_t usedVGPRS = 0;
size_t usedSGPRS = 0;
size_t usedLDS = 0;
getGprsLdsUsage(f, &usedVGPRS, &usedSGPRS, &usedLDS);
// Due to SPI and private memory limitations, the max of wavefronts per CU in 32
size_t wavefrontSize = prop.warpSize;
size_t maxWavefrontsPerCU = min(prop.maxThreadsPerMultiProcessor / wavefrontSize, 32);
const size_t simdPerCU = 4;
const size_t maxWavesPerSimd = maxWavefrontsPerCU / simdPerCU;
size_t numWavefronts = (blockSize + wavefrontSize - 1) / wavefrontSize;
size_t availableVGPRs = (prop.regsPerBlock / wavefrontSize / simdPerCU);
size_t vgprs_alu_occupancy = simdPerCU * (usedVGPRS == 0 ? maxWavesPerSimd
: std::min(maxWavesPerSimd, availableVGPRs / usedVGPRS));
// Calculate blocks occupancy per CU based on VGPR usage
*numBlocks = vgprs_alu_occupancy / numWavefronts;
const size_t availableSGPRs = (prop.gcnArch < 800) ? 512 : 800;
size_t sgprs_alu_occupancy = simdPerCU * (usedSGPRS == 0 ? maxWavesPerSimd
: std::min(maxWavesPerSimd, availableSGPRs / usedSGPRS));
// Calculate blocks occupancy per CU based on SGPR usage
*numBlocks = std::min(*numBlocks, (uint32_t) (sgprs_alu_occupancy / numWavefronts));
size_t total_used_lds = usedLDS + dynSharedMemPerBlk;
if (total_used_lds != 0) {
// Calculate LDS occupacy per CU. lds_per_cu / (static_lsd + dynamic_lds)
size_t lds_occupancy = prop.maxSharedMemoryPerMultiProcessor / total_used_lds;
*numBlocks = std::min(*numBlocks, (uint32_t) lds_occupancy);
}
return hipSuccess;
}
namespace {
// kernel for initializing GWS
// nwm1 is the total number of work groups minus 1
@@ -417,25 +499,28 @@ __global__ void init_gws(uint nwm1) {
}
}
__attribute__((visibility("default")))
hipError_t ihipLaunchCooperativeKernel(const void* f, dim3 gridDim,
dim3 blockDimX, void** kernelParams, unsigned int sharedMemBytes,
dim3 blockDim, void** kernelParams, unsigned int sharedMemBytes,
hipStream_t stream, hip_impl::program_state& ps) {
#if (__hcc_workweek__ >= 20093)
hipError_t result;
if ((f == nullptr) || (stream == nullptr) || (kernelParams == nullptr)) {
if (f == nullptr || kernelParams == nullptr) {
return hipErrorNotInitialized;
}
if (!stream->getDevice()->_props.cooperativeLaunch) {
stream = ihipSyncAndResolveStream(stream);
if (!stream->getDevice()->_props.cooperativeLaunch ||
blockDim.x * blockDim.y * blockDim.z > stream->getDevice()->_props.maxThreadsPerBlock) {
return hipErrorInvalidConfiguration;
}
size_t globalWorkSizeX = (size_t)gridDim.x * (size_t)blockDimX.x;
size_t globalWorkSizeY = (size_t)gridDim.y * (size_t)blockDimX.y;
size_t globalWorkSizeZ = (size_t)gridDim.z * (size_t)blockDimX.z;
size_t globalWorkSizeX = (size_t)gridDim.x * (size_t)blockDim.x;
size_t globalWorkSizeY = (size_t)gridDim.y * (size_t)blockDim.y;
size_t globalWorkSizeZ = (size_t)gridDim.z * (size_t)blockDim.z;
if(globalWorkSizeX > UINT32_MAX || globalWorkSizeY > UINT32_MAX || globalWorkSizeZ > UINT32_MAX)
{
return hipErrorInvalidConfiguration;
@@ -469,28 +554,49 @@ hipError_t ihipLaunchCooperativeKernel(const void* f, dim3 gridDim,
kd->_kernarg_layout = *reinterpret_cast<const std::vector<
std::pair<std::size_t, std::size_t>>*>(kargs.getHandle());
GET_TLS();
uint32_t numBlocksPerSm = 0;
result = ihipOccupancyMaxActiveBlocksPerMultiprocessor(tls, &numBlocksPerSm, kd,
blockDim.x * blockDim.y * blockDim.z, sharedMemBytes);
if (result != hipSuccess) {
return hipErrorLaunchFailure;
}
int maxActiveBlocks = numBlocksPerSm * stream->getDevice()->_props.multiProcessorCount;
//check to see if the workload fits on the GPU
if (gridDim.x * gridDim.y * gridDim.z > maxActiveBlocks) {
return hipErrorCooperativeLaunchTooLarge;
}
void *gwsKernelParam[1];
// calculate total number of work groups minus 1 for the main kernel
uint nwm1 = (gridDim.x * gridDim.y * gridDim.z) - 1;
gwsKernelParam[0] = &nwm1;
LockedAccessor_StreamCrit_t streamCrit(stream->criticalData(), false);
#if (__hcc_workweek__ >= 19213)
streamCrit->_av.acquire_locked_hsa_queue();
#endif
hc::accelerator acc = stream->getDevice()->_acc;
// create a cooperative accelerated view for launching gws and main kernels
hc::accelerator_view coopAV = acc.create_cooperative_view();
GET_TLS();
// launch the init_gws kernel to initialize the GWS
LockedAccessor_StreamCrit_t streamCrit(stream->criticalData(), false);
// the cooperative queue will wait until this stream completes its operations
hc::completion_future streamCF;
if (!streamCrit->_av.get_is_empty()) {
streamCF = streamCrit->_av.create_marker(hc::accelerator_scope);
coopAV.create_blocking_marker(streamCF, hc::accelerator_scope);
}
streamCrit->_av.acquire_locked_hsa_queue();
coopAV.acquire_locked_hsa_queue();
// launch the init_gws kernel to initialize the GWS in the dedicated cooperative queue
result = ihipModuleLaunchKernel(tls, gwsKD, 1, 1, 1, 1, 1, 1,
0, stream, gwsKernelParam, nullptr, nullptr, nullptr, 0, true);
0, stream, gwsKernelParam, nullptr, nullptr, nullptr, 0, true, nullptr , &coopAV);
if (result != hipSuccess) {
stream->criticalData().unlock();
#if (__hcc_workweek__ >= 19213)
stream->criticalData()._av.release_locked_hsa_queue();
#endif
coopAV.release_locked_hsa_queue();
return hipErrorLaunchFailure;
}
@@ -498,60 +604,106 @@ hipError_t ihipLaunchCooperativeKernel(const void* f, dim3 gridDim,
void* impCoopParams[1];
impCoopParams[0] = &impCoopArg;
// launch the main kernel
// launch the main kernel in the cooperative queue
result = ihipModuleLaunchKernel(tls, kd,
gridDim.x * blockDimX.x,
gridDim.y * blockDimX.y,
gridDim.z * blockDimX.z,
blockDimX.x, blockDimX.y, blockDimX.z,
gridDim.x * blockDim.x,
gridDim.y * blockDim.y,
gridDim.z * blockDim.z,
blockDim.x, blockDim.y, blockDim.z,
sharedMemBytes, stream, kernelParams, nullptr, nullptr,
nullptr, 0, true, impCoopParams);
nullptr, 0, true, impCoopParams, &coopAV);
coopAV.release_locked_hsa_queue();
stream->criticalData()._av.release_locked_hsa_queue();
// this stream will wait until the cooperative queue completes its operations
hc::completion_future cooperativeCF;
if (!coopAV.get_is_empty()) {
cooperativeCF = coopAV.create_marker(hc::accelerator_scope);
streamCrit->_av.create_blocking_marker(cooperativeCF, hc::accelerator_scope);
}
stream->criticalData().unlock();
#if (__hcc_workweek__ >= 19213)
stream->criticalData()._av.release_locked_hsa_queue();
#endif
return result;
#else
return hipErrorInvalidConfiguration;
#endif
}
__attribute__((visibility("default")))
hipError_t hipLaunchCooperativeKernel(const void* func, dim3 gridDim,
dim3 blockDim, void** args,
size_t sharedMem, hipStream_t stream,
hip_impl::program_state& ps) {
// Skipping passing in ps, because the logging function does not like it
HIP_INIT_API(hipLaunchCooperativeKernel, func, gridDim, blockDim, args,
sharedMem, stream);
return ihipLogStatus(ihipLaunchCooperativeKernel(func, gridDim, blockDim,
args, sharedMem, stream, ps));
}
hipError_t ihipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsList,
int numDevices, unsigned int flags, hip_impl::program_state& ps) {
#if (__hcc_workweek__ >= 20093)
hipError_t result;
if (numDevices > g_deviceCnt || launchParamsList == nullptr || numDevices > MAX_COOPERATIVE_GPUs) {
return hipErrorInvalidValue;
}
vector<hipStream_t> streams;
vector<uint64_t> deviceIDs;
// check to see if we have valid distinct streams/devices, if cooperative multi device
// launch is supported and if grid/block dimensions are valid
for (int i = 0; i < numDevices; ++i) {
if (!launchParamsList[i].stream->getDevice()->_props.cooperativeMultiDeviceLaunch) {
const hipLaunchParams& lp = launchParamsList[i];
if (lp.stream == nullptr){
return hipErrorInvalidResourceHandle;
}
if (find(streams.begin(), streams.end(), lp.stream) == streams.end()) {
streams.push_back(lp.stream);
} else {
return hipErrorInvalidDevice;
}
const ihipDevice_t* currentDevice = lp.stream->getDevice();
if (find(deviceIDs.begin(), deviceIDs.end(), currentDevice->_deviceId) == deviceIDs.end()) {
deviceIDs.push_back(currentDevice->_deviceId);
} else {
return hipErrorInvalidDevice;
}
if (!currentDevice->_props.cooperativeMultiDeviceLaunch) {
return hipErrorInvalidConfiguration;
}
if (lp.gridDim.x == 0 || lp.gridDim.y == 0 || lp.gridDim.z == 0 ||
lp.blockDim.x == 0 || lp.blockDim.y == 0 || lp.blockDim.z == 0 ||
lp.blockDim.x * lp.blockDim.y * lp.blockDim.z > currentDevice->_props.maxThreadsPerBlock){
return hipErrorInvalidConfiguration;
}
}
hipFunction_t* gwsKds = reinterpret_cast<hipFunction_t*>(malloc(sizeof(hipFunction_t) * numDevices));
hipFunction_t* kds = reinterpret_cast<hipFunction_t*>(malloc(sizeof(hipFunction_t) * numDevices));
if (kds == nullptr || gwsKds == nullptr) {
return hipErrorNotInitialized;
}
vector<hipFunction_t> gwsKds;
vector<hipFunction_t> kds;
GET_TLS();
// prepare all kernel descriptors for initializing the GWS and the main kernels per device
for (int i = 0; i < numDevices; ++i) {
const hipLaunchParams& lp = launchParamsList[i];
if (lp.stream == nullptr) {
free(gwsKds);
free(kds);
return hipErrorNotInitialized;
}
gwsKds[i] = ps.kernel_descriptor(reinterpret_cast<std::uintptr_t>(&init_gws),
hip_impl::target_agent(lp.stream));
gwsKds.push_back(ps.kernel_descriptor(reinterpret_cast<std::uintptr_t>(&init_gws),
hip_impl::target_agent(lp.stream)));
if (gwsKds[i] == nullptr) {
free(gwsKds);
free(kds);
return hipErrorInvalidValue;
}
hip_impl::kernargs_size_align gwsKargs = ps.get_kernargs_size_align(
@@ -560,23 +712,42 @@ hipError_t ihipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsL
gwsKargs.getHandle());
kds[i] = ps.kernel_descriptor(reinterpret_cast<std::uintptr_t>(lp.func),
hip_impl::target_agent(lp.stream));
kds.push_back(ps.kernel_descriptor(reinterpret_cast<std::uintptr_t>(lp.func),
hip_impl::target_agent(lp.stream)));
if (kds[i] == nullptr) {
free(gwsKds);
free(kds);
return hipErrorInvalidValue;
}
hip_impl::kernargs_size_align kargs = ps.get_kernargs_size_align(
reinterpret_cast<std::uintptr_t>(lp.func));
kds[i]->_kernarg_layout = *reinterpret_cast<const std::vector<std::pair<std::size_t, std::size_t>>*>(
kargs.getHandle());
uint32_t numBlocksPerSm = 0;
result = ihipOccupancyMaxActiveBlocksPerMultiprocessor(tls, &numBlocksPerSm, kds[i],
lp.blockDim.x * lp.blockDim.y * lp.blockDim.z, lp.sharedMem);
if (result != hipSuccess) {
return hipErrorLaunchFailure;
}
int maxActiveBlocks = numBlocksPerSm * lp.stream->getDevice()->_props.multiProcessorCount;
//check to see if the workload fits on the GPU
if (lp.gridDim.x * lp.gridDim.y * lp.gridDim.z > maxActiveBlocks) {
return hipErrorCooperativeLaunchTooLarge;
}
}
vector<hc::accelerator_view> coopAVs;
// create cooperative accelerated views for launching gws and main kernels on each device
for (int i = 0; i < numDevices; ++i) {
hc::accelerator acc = launchParamsList[i].stream->getDevice()->_acc;
coopAVs.push_back(acc.create_cooperative_view());
}
mg_sync *mg_sync_ptr = 0;
mg_info *mg_info_ptr[MAX_COOPERATIVE_GPUs] = {0};
vector<mg_info *> mg_info_ptr;
GET_TLS();
result = hip_internal::ihipHostMalloc(tls, (void **)&mg_sync_ptr, sizeof(mg_sync), hipHostMallocDefault);
if (result != hipSuccess) {
return hipErrorInvalidValue;
@@ -586,7 +757,8 @@ hipError_t ihipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsL
uint all_sum = 0;
for (int i = 0; i < numDevices; ++i) {
result = hip_internal::ihipHostMalloc(tls, (void **)&mg_info_ptr[i], sizeof(mg_info), hipHostMallocDefault);
mg_info *mg_info_temp = nullptr;
result = hip_internal::ihipHostMalloc(tls, (void **)&mg_info_temp, sizeof(mg_info), hipHostMallocDefault);
if (result != hipSuccess) {
hip_internal::ihipHostFree(tls, mg_sync_ptr);
for (int j = 0; j < i; ++j) {
@@ -594,6 +766,7 @@ hipError_t ihipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsL
}
return hipErrorInvalidValue;
}
mg_info_ptr.push_back(mg_info_temp);
// calculate the sum of sizes of all grids
const hipLaunchParams& lp = launchParamsList[i];
all_sum += lp.blockDim.x * lp.blockDim.y * lp.blockDim.z *
@@ -603,9 +776,15 @@ hipError_t ihipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsL
// lock all streams before launching the blit kernels for initializing the GWS and main kernels to each device
for (int i = 0; i < numDevices; ++i) {
LockedAccessor_StreamCrit_t streamCrit(launchParamsList[i].stream->criticalData(), false);
#if (__hcc_workweek__ >= 19213)
hc::completion_future streamCF;
if (!streamCrit->_av.get_is_empty()) {
streamCF = streamCrit->_av.create_marker(hc::accelerator_scope);
coopAVs[i].create_blocking_marker(streamCF, hc::accelerator_scope);
}
streamCrit->_av.acquire_locked_hsa_queue();
#endif
coopAVs[i].acquire_locked_hsa_queue();
}
// launch the init_gws kernel to initialize the GWS for each device
@@ -617,14 +796,13 @@ hipError_t ihipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsL
gwsKernelParam[0] = &nwm1;
result = ihipModuleLaunchKernel(tls, gwsKds[i], 1, 1, 1, 1, 1, 1,
0, lp.stream, gwsKernelParam, nullptr, nullptr, nullptr, 0, true);
0, lp.stream, gwsKernelParam, nullptr, nullptr, nullptr, 0, true, nullptr, &coopAVs[i]);
if (result != hipSuccess) {
for (int j = 0; j < numDevices; ++j) {
launchParamsList[j].stream->criticalData().unlock();
#if (__hcc_workweek__ >= 19213)
launchParamsList[j].stream->criticalData()._av.release_locked_hsa_queue();
#endif
coopAVs[i].release_locked_hsa_queue();
}
hip_internal::ihipHostFree(tls, mg_sync_ptr);
for (int j = 0; j < numDevices; ++j) {
@@ -670,14 +848,13 @@ hipError_t ihipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsL
lp.blockDim.x, lp.blockDim.y,
lp.blockDim.z, lp.sharedMem,
lp.stream, lp.args, nullptr, nullptr, nullptr, 0,
true, impCoopParams);
true, impCoopParams, &coopAVs[i]);
if (result != hipSuccess) {
for (int j = 0; j < numDevices; ++j) {
launchParamsList[j].stream->criticalData().unlock();
#if (__hcc_workweek__ >= 19213)
launchParamsList[j].stream->criticalData()._av.release_locked_hsa_queue();
#endif
coopAVs[i].release_locked_hsa_queue();
}
hip_internal::ihipHostFree(tls, mg_sync_ptr);
for (int j = 0; j < numDevices; ++j) {
@@ -691,14 +868,18 @@ hipError_t ihipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsL
// unlock all streams
for (int i = 0; i < numDevices; ++i) {
launchParamsList[i].stream->criticalData().unlock();
#if (__hcc_workweek__ >= 19213)
coopAVs[i].release_locked_hsa_queue();
launchParamsList[i].stream->criticalData()._av.release_locked_hsa_queue();
#endif
}
free(gwsKds);
free(kds);
hc::completion_future cooperativeCF;
if (!coopAVs[i].get_is_empty()) {
cooperativeCF = coopAVs[i].create_marker(hc::accelerator_scope);
launchParamsList[i].stream->criticalData()._av.create_blocking_marker(
cooperativeCF, hc::accelerator_scope);
}
launchParamsList[i].stream->criticalData().unlock();
}
hip_internal::ihipHostFree(tls, mg_sync_ptr);
for (int j = 0; j < numDevices; ++j) {
@@ -706,6 +887,24 @@ hipError_t ihipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsL
}
return result;
#else
return hipErrorInvalidConfiguration;
#endif
}
__attribute__((visibility("default")))
hipError_t hipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsList,
int numDevices,
unsigned int flags,
hip_impl::program_state& ps) {
// Skipping passing in ps, because the logging function does not like it
HIP_INIT_API(hipLaunchCooperativeKernelMultiDevice, launchParamsList,
numDevices, flags);
return ihipLogStatus(ihipLaunchCooperativeKernelMultiDevice(launchParamsList,
numDevices,
flags, ps));
}
namespace hip_impl {
@@ -1120,7 +1319,7 @@ const amd_kernel_code_v3_t *header_v3(const ihipModuleSymbol_t& kd) {
return reinterpret_cast<const amd_kernel_code_v3_t*>(kd._header);
}
hipFuncAttributes make_function_attributes(TlsData *tls, const ihipModuleSymbol_t& kd) {
hipFuncAttributes make_function_attributes(TlsData *tls, ihipModuleSymbol_t& kd) {
hipFuncAttributes r{};
hipDeviceProp_t prop{};
@@ -1130,23 +1329,57 @@ hipFuncAttributes make_function_attributes(TlsData *tls, const ihipModuleSymbol_
prop.regsPerBlock = prop.regsPerBlock ? prop.regsPerBlock : 64 * 1024;
if (kd._is_code_object_v3) {
r.localSizeBytes = header_v3(kd)->private_segment_fixed_size;
r.sharedSizeBytes = header_v3(kd)->group_segment_fixed_size;
r.numRegs = ((header_v3(kd)->compute_pgm_rsrc1 & 0x3F) + 1) << 2;
r.binaryVersion = 0; // FIXME: should it be the ISA version or code
// object format version?
r.localSizeBytes = header_v3(kd)->private_segment_fixed_size;
r.sharedSizeBytes = header_v3(kd)->group_segment_fixed_size;
} else {
r.localSizeBytes = kd._header->workitem_private_segment_byte_size;
r.sharedSizeBytes = kd._header->workgroup_group_segment_byte_size;
r.numRegs = kd._header->workitem_vgpr_count;
r.binaryVersion =
kd._header->amd_machine_version_major * 10 +
kd._header->amd_machine_version_minor;
}
r.maxDynamicSharedSizeBytes = prop.sharedMemPerBlock - r.sharedSizeBytes;
r.maxThreadsPerBlock = r.numRegs ?
std::min(prop.maxThreadsPerBlock, prop.regsPerBlock / r.numRegs) :
prop.maxThreadsPerBlock;
size_t usedVGPRS = 0;
size_t usedSGPRS = 0;
size_t usedLDS = 0;
getGprsLdsUsage(&kd, &usedVGPRS, &usedSGPRS, &usedLDS);
r.numRegs = usedVGPRS;
size_t wavefrontSize = prop.warpSize;
size_t maxWavefrontsPerBlock = prop.maxThreadsPerBlock / wavefrontSize;
size_t maxWavefrontsPerCU = min(prop.maxThreadsPerMultiProcessor / wavefrontSize, 32);
const size_t numSIMD = 4;
const size_t maxWavesPerSimd = maxWavefrontsPerCU / numSIMD;
size_t maxWaves = 0;
for (int i = 0; i < maxWavefrontsPerBlock; i++) {
size_t wavefronts = i + 1;
if (usedVGPRS > 0) {
size_t availableVGPRs = (prop.regsPerBlock / wavefrontSize / numSIMD);
size_t vgprs_alu_occupancy = numSIMD * std::min(maxWavesPerSimd, availableVGPRs / usedVGPRS);
// Calculate blocks occupancy per CU based on VGPR usage
if (vgprs_alu_occupancy < wavefronts)
break;
}
if (usedSGPRS > 0) {
const size_t availableSGPRs = (prop.gcnArch < 800) ? 512 : 800;
size_t sgprs_alu_occupancy = numSIMD * ((usedSGPRS == 0) ? maxWavesPerSimd
: std::min(maxWavesPerSimd, availableSGPRs / usedSGPRS));
// Calculate blocks occupancy per CU based on SGPR usage
if (sgprs_alu_occupancy < wavefronts)
break;
}
maxWaves = wavefronts;
}
r.maxThreadsPerBlock = maxWaves * wavefrontSize;
r.ptxVersion = prop.major * 10 + prop.minor; // HIP currently presents itself as PTX 3.0.
return r;
@@ -1294,29 +1527,6 @@ hipError_t hipModuleGetTexRef(textureReference** texRef, hipModule_t hmod, const
return ihipLogStatus(hipSuccess);
}
void getGprsLdsUsage(hipFunction_t f, size_t* usedVGPRS, size_t* usedSGPRS, size_t* usedLDS)
{
if (f->_is_code_object_v3) {
const auto header = reinterpret_cast<const amd_kernel_code_v3_t*>(f->_header);
// GRANULATED_WAVEFRONT_VGPR_COUNT is specified in 0:5 bits of COMPUTE_PGM_RSRC1
// the granularity for gfx6-gfx9 is max(0, ceil(vgprs_used / 4) - 1)
*usedVGPRS = ((header->compute_pgm_rsrc1 & 0x3F) + 1) << 2;
// GRANULATED_WAVEFRONT_SGPR_COUNT is specified in 6:9 bits of COMPUTE_PGM_RSRC1
// the granularity for gfx9+ is 2 * max(0, ceil(sgprs_used / 16) - 1)
*usedSGPRS = ((((header->compute_pgm_rsrc1 & 0x3C0) >> 6) >> 1) + 1) << 4;
*usedLDS = header->group_segment_fixed_size;
}
else {
const auto header = f->_header;
// VGPRs granularity is 4
*usedVGPRS = ((header->workitem_vgpr_count + 3) >> 2) << 2;
// adding 2 to take into account the 2 VCC registers & handle the granularity of 16
*usedSGPRS = header->wavefront_sgpr_count + 2;
*usedSGPRS = ((*usedSGPRS + 15) >> 4) << 4;
*usedLDS = header->workgroup_group_segment_byte_size;
}
}
hipError_t ihipOccupancyMaxPotentialBlockSize(TlsData *tls, uint32_t* gridSize, uint32_t* blockSize,
hipFunction_t f, size_t dynSharedMemPerBlk,
uint32_t blockSizeLimit)
@@ -1439,59 +1649,6 @@ hipError_t hipOccupancyMaxPotentialBlockSize(uint32_t* gridSize, uint32_t* block
gridSize, blockSize, f, dynSharedMemPerBlk, blockSizeLimit));
}
hipError_t ihipOccupancyMaxActiveBlocksPerMultiprocessor(
TlsData *tls, uint32_t* numBlocks, hipFunction_t f, uint32_t blockSize, size_t dynSharedMemPerBlk)
{
using namespace hip_impl;
auto ctx = ihipGetTlsDefaultCtx();
if (ctx == nullptr) {
return hipErrorInvalidDevice;
}
hipDeviceProp_t prop{};
ihipGetDeviceProperties(&prop, ihipGetTlsDefaultCtx()->getDevice()->_deviceId);
prop.regsPerBlock = prop.regsPerBlock ? prop.regsPerBlock : 64 * 1024;
size_t usedVGPRS = 0;
size_t usedSGPRS = 0;
size_t usedLDS = 0;
getGprsLdsUsage(f, &usedVGPRS, &usedSGPRS, &usedLDS);
// Due to SPI and private memory limitations, the max of wavefronts per CU in 32
size_t wavefrontSize = prop.warpSize;
size_t maxWavefrontsPerCU = min(prop.maxThreadsPerMultiProcessor / wavefrontSize, 32);
const size_t simdPerCU = 4;
const size_t maxWavesPerSimd = maxWavefrontsPerCU / simdPerCU;
size_t numWavefronts = (blockSize + wavefrontSize - 1) / wavefrontSize;
size_t availableVGPRs = (prop.regsPerBlock / wavefrontSize / simdPerCU);
size_t vgprs_alu_occupancy = simdPerCU * (usedVGPRS == 0 ? maxWavesPerSimd
: std::min(maxWavesPerSimd, availableVGPRs / usedVGPRS));
// Calculate blocks occupancy per CU based on VGPR usage
*numBlocks = vgprs_alu_occupancy / numWavefronts;
const size_t availableSGPRs = (prop.gcnArch < 800) ? 512 : 800;
size_t sgprs_alu_occupancy = simdPerCU * (usedSGPRS == 0 ? maxWavesPerSimd
: std::min(maxWavesPerSimd, availableSGPRs / usedSGPRS));
// Calculate blocks occupancy per CU based on SGPR usage
*numBlocks = std::min(*numBlocks, (uint32_t) (sgprs_alu_occupancy / numWavefronts));
size_t total_used_lds = usedLDS + dynSharedMemPerBlk;
if (total_used_lds != 0) {
// Calculate LDS occupacy per CU. lds_per_cu / (static_lsd + dynamic_lds)
size_t lds_occupancy = prop.maxSharedMemoryPerMultiProcessor / total_used_lds;
*numBlocks = std::min(*numBlocks, (uint32_t) lds_occupancy);
}
return hipSuccess;
}
hipError_t hipOccupancyMaxActiveBlocksPerMultiprocessor(
uint32_t* numBlocks, hipFunction_t f, uint32_t blockSize, size_t dynSharedMemPerBlk)
{
projects/hip/src/hip_stream.cpp
+33 -5
View File
@@ -257,11 +257,39 @@ hipError_t hipStreamGetPriority(hipStream_t stream, int* priority) {
hipError_t hipStreamAddCallback(hipStream_t stream, hipStreamCallback_t callback, void* userData,
unsigned int flags) {
HIP_INIT_API(hipStreamAddCallback, stream, callback, userData, flags);
hipError_t e = hipSuccess;
// Create a thread in detached mode to handle callback
ihipStreamCallback_t* cb = new ihipStreamCallback_t(stream, callback, userData);
std::thread(ihipStreamCallbackHandler, cb).detach();
auto stream_original{stream};
stream = ihipSyncAndResolveStream(stream);
return ihipLogStatus(e);
if (!stream) return hipErrorInvalidValue;
LockedAccessor_StreamCrit_t cs{stream->criticalData()};
// create first marker
auto cf = cs->_av.create_marker(hc::no_scope);
// get its signal
auto signal = *reinterpret_cast<hsa_signal_t*>(cf.get_native_handle());
// increment its signal value
hsa_signal_add_relaxed(signal, 1);
// create callback that can be passed to hsa_amd_signal_async_handler
// this function will call the user's callback, then sets first packet's signal to 0 to indicate completion
auto t{new std::function<void()>{[=]() {
callback(stream_original, hipSuccess, userData);
hsa_signal_store_relaxed(signal, 0);
}}};
// register above callback with HSA runtime to be called when first packet's signal
// is decremented from 2 to 1 by CP (or it is already at 1)
hsa_amd_signal_async_handler(signal, HSA_SIGNAL_CONDITION_EQ, 1,
[](hsa_signal_value_t x, void* p) {
(*static_cast<decltype(t)>(p))();
delete static_cast<decltype(t)>(p);
return false;
}, t);
// create additional marker that blocks on the first one
cs->_av.create_blocking_marker(cf, hc::no_scope);
return ihipLogStatus(hipSuccess);
}
projects/hip/src/hip_texture.cpp
+22 -4
View File
@@ -301,7 +301,12 @@ hipError_t hipCreateTextureObject(hipTextureObject_t* pTexObject, const hipResou
hsa_ext_sampler_descriptor_t samplerDescriptor;
fillSamplerDescriptor(samplerDescriptor, pTexDesc->addressMode[0], pTexDesc->filterMode,
pTexDesc->normalizedCoords);
if(hipResourceTypeLinear == pResDesc->resType) {
samplerDescriptor.filter_mode = HSA_EXT_SAMPLER_FILTER_MODE_NEAREST;
samplerDescriptor.address_mode = HSA_EXT_SAMPLER_ADDRESSING_MODE_CLAMP_TO_BORDER;
} else if(!pTexDesc->normalizedCoords) {
samplerDescriptor.address_mode = HSA_EXT_SAMPLER_ADDRESSING_MODE_CLAMP_TO_EDGE;
}
hsa_access_permission_t permission = HSA_ACCESS_PERMISSION_RW;
if(hipResourceTypePitch2D != pResDesc->resType)
@@ -312,6 +317,7 @@ hipError_t hipCreateTextureObject(hipTextureObject_t* pTexObject, const hipResou
HSA_EXT_IMAGE_DATA_LAYOUT_LINEAR, pitch, 0, &(pTexture->image)) ||
HSA_STATUS_SUCCESS !=
hsa_ext_sampler_create(*agent, &samplerDescriptor, &(pTexture->sampler))) {
free(pTexture);
return ihipLogStatus(hipErrorRuntimeOther);
}
@@ -438,7 +444,13 @@ hipError_t ihipBindTextureImpl(TlsData *tls_, int dim, enum hipTextureReadMode r
imageDescriptor.format.channel_type = channelType;
hsa_ext_sampler_descriptor_t samplerDescriptor;
fillSamplerDescriptor(samplerDescriptor, addressMode, filterMode, normalizedCoords);
samplerDescriptor.filter_mode = HSA_EXT_SAMPLER_FILTER_MODE_NEAREST;
samplerDescriptor.address_mode = HSA_EXT_SAMPLER_ADDRESSING_MODE_CLAMP_TO_BORDER;
if (normalizedCoords) {
samplerDescriptor.coordinate_mode = HSA_EXT_SAMPLER_COORDINATE_MODE_NORMALIZED;
} else {
samplerDescriptor.coordinate_mode = HSA_EXT_SAMPLER_COORDINATE_MODE_UNNORMALIZED;
}
hsa_access_permission_t permission = HSA_ACCESS_PERMISSION_RW;
@@ -449,6 +461,7 @@ hipError_t ihipBindTextureImpl(TlsData *tls_, int dim, enum hipTextureReadMode r
HSA_EXT_IMAGE_DATA_LAYOUT_LINEAR, rowPitch, 0, &(pTexture->image)) ||
HSA_STATUS_SUCCESS !=
hsa_ext_sampler_create(*agent, &samplerDescriptor, &(pTexture->sampler))) {
free(pTexture);
return hipErrorRuntimeOther;
}
getHipTextureObject(&textureObject, pTexture->image, pTexture->sampler);
@@ -514,7 +527,9 @@ hipError_t ihipBindTexture2DImpl(TlsData *tls, int dim, enum hipTextureReadMode
hsa_ext_sampler_descriptor_t samplerDescriptor;
fillSamplerDescriptor(samplerDescriptor, addressMode, filterMode, normalizedCoords);
if(!normalizedCoords) {
samplerDescriptor.address_mode = HSA_EXT_SAMPLER_ADDRESSING_MODE_CLAMP_TO_EDGE;
}
hsa_access_permission_t permission = HSA_ACCESS_PERMISSION_RW;
if( 0 == pitch)
@@ -525,6 +540,7 @@ hipError_t ihipBindTexture2DImpl(TlsData *tls, int dim, enum hipTextureReadMode
HSA_EXT_IMAGE_DATA_LAYOUT_LINEAR, pitch, 0, &(pTexture->image)) ||
HSA_STATUS_SUCCESS !=
hsa_ext_sampler_create(*agent, &samplerDescriptor, &(pTexture->sampler))) {
free(pTexture);
return hipErrorRuntimeOther;
}
getHipTextureObject(&textureObject, pTexture->image, pTexture->sampler);
@@ -620,7 +636,9 @@ hipError_t ihipBindTextureToArrayImpl(TlsData *tls_, int dim, enum hipTextureRea
hsa_ext_sampler_descriptor_t samplerDescriptor;
fillSamplerDescriptor(samplerDescriptor, addressMode, filterMode, normalizedCoords);
if(!normalizedCoords) {
samplerDescriptor.address_mode = HSA_EXT_SAMPLER_ADDRESSING_MODE_CLAMP_TO_EDGE;
}
hsa_access_permission_t permission = HSA_ACCESS_PERMISSION_RW;
size_t rowPitch = getElementSize(channelOrder, channelType) * alignUp(imageDescriptor.width, IMAGE_PITCH_ALIGNMENT);
projects/hip/src/hiprtc.cpp
+1 -1
View File
@@ -21,7 +21,7 @@ THE SOFTWARE.
*/
#include "../include/hip/hiprtc.h"
#include "../include/hip/hcc_detail/code_object_bundle.hpp"
#include "code_object_bundle.inl"
#include "../include/hip/hcc_detail/elfio/elfio.hpp"
#include "../include/hip/hcc_detail/program_state.hpp"
projects/hip/src/program_state.inl
+6 -3
View File
@@ -1,6 +1,6 @@
#include "../include/hip/hcc_detail/program_state.hpp"
#include "../include/hip/hcc_detail/code_object_bundle.hpp"
#include "code_object_bundle.inl"
#include "../include/hip/hcc_detail/hsa_helpers.hpp"
#if !defined(__cpp_exceptions)
@@ -357,8 +357,11 @@ public:
const auto it1 = get_symbol_addresses().find(x);
if (it1 == get_symbol_addresses().cend()) {
hip_throw(std::runtime_error{
"Global symbol: " + x + " is undefined."});
// For a unknown symbol, initialize it with a magic poison
hsa_executable_agent_global_variable_define(
executable, agent, x.c_str(),
reinterpret_cast<void*>(0xDEADBEEFDEADBEEFull));
continue;
}
hsa_status_t status;