RCCL support (#93)
* Initial support for RCCL * OMNITRACE_USE_RCCLP + sampling tweaks - also OMNITRACE_SAMPLING_KEEP_INTERNAL option - minor modifications to sampling to use keep internal option + discard funlockfile * Update docker and workflows to download RCCL * Update CPack DEB with rocprofiler dependency * Rework rccl into library and library/components folder - add tpls/rccl/rccl/rccl.h * Fix timemory includes * rcclp inline definitions when disabled * Tweaks to ubuntu-focal-external-rocm - disable ompt - enable building testing * Tweaks to ubuntu-focal-external-rocm - ctest exclude * Tweak ubuntu-focal.yml - remove source /.../setup-env.sh, replace with $GITHUB_ENV * Fix ubuntu-focal-rocm + OMPI + root * Improved rocm-smi error handling - Recover from rocm-smi errors - Disabling rocm-smi after recovering from errors - Werror in developer mode - Remove State::DelayedInit - Add State::Disabled * formatting * Fix merge of OMNITRACE_SAMPLING_KEEP_INTERNAL * Update RCCL include directory - based on ROCm version we need with <rccl/rccl.h> or <rccl.h> * RCCL Testing - updated tests to use configuration files - many tests generate a configuration file - tests how have GPU option - enable ncclCommCount, disable ncclGetVersion - add testing for RCCLP via rccl-tests - working directory of tests is PROJECT_BINARY_DIR - add nccl/rccl functions to get_whole_function_names - some clang compiler fixes * Handle RCCL include w/o HIP * RCCL requires HIP * Update OMNITRACE_SAMPLING_CPUS for testing * Update tests/CMakeLists.txt * Debug settings * Install MPI even when USE_MPI=OFF * exclude printf * skip mpi tests w/o USE_MPI or USE_MPI_HEADERS * update ubuntu rocm workflow * Fix configure env step for ubuntu rocm
This commit is contained in:
Jonathan R. Madsen
committed by
GitHub
GitHub
parent
a539da1681
commit
45be03906a
@@ -0,0 +1,269 @@
|
||||
// MIT License
|
||||
//
|
||||
// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
|
||||
#include "library/components/rcclp.hpp"
|
||||
#include "library/rcclp.hpp"
|
||||
|
||||
#include <timemory/manager.hpp>
|
||||
|
||||
std::ostream&
|
||||
operator<<(std::ostream& _os, const ncclUniqueId& _v)
|
||||
{
|
||||
for(auto itr : _v.internal)
|
||||
_os << itr;
|
||||
return _os;
|
||||
}
|
||||
|
||||
namespace tim
|
||||
{
|
||||
namespace component
|
||||
{
|
||||
uint64_t
|
||||
activate_rcclp()
|
||||
{
|
||||
using handle_t = tim::component::rcclp_handle;
|
||||
|
||||
static auto _handle = std::shared_ptr<handle_t>{};
|
||||
|
||||
if(!_handle.get())
|
||||
{
|
||||
_handle = std::make_shared<handle_t>();
|
||||
_handle->start();
|
||||
|
||||
auto cleanup_functor = [=]() {
|
||||
if(_handle)
|
||||
{
|
||||
_handle->stop();
|
||||
_handle.reset();
|
||||
}
|
||||
};
|
||||
|
||||
std::stringstream ss;
|
||||
ss << "timemory-rcclp-" << demangle<rccl_toolset_t>() << "-"
|
||||
<< demangle<api::rccl>();
|
||||
tim::manager::instance()->add_cleanup(ss.str(), cleanup_functor);
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
//
|
||||
//======================================================================================//
|
||||
//
|
||||
uint64_t
|
||||
deactivate_rcclp(uint64_t id)
|
||||
{
|
||||
if(id > 0)
|
||||
{
|
||||
std::stringstream ss;
|
||||
ss << "timemory-rcclp-" << demangle<rccl_toolset_t>() << "-"
|
||||
<< demangle<api::rccl>();
|
||||
tim::manager::instance()->cleanup(ss.str());
|
||||
return 0;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
//
|
||||
//======================================================================================//
|
||||
//
|
||||
void
|
||||
configure_rcclp(const std::set<std::string>& permit, const std::set<std::string>& reject)
|
||||
{
|
||||
static constexpr size_t rcclp_wrapper_count = OMNITRACE_NUM_RCCLP_WRAPPERS;
|
||||
|
||||
using rcclp_gotcha_t =
|
||||
tim::component::gotcha<rcclp_wrapper_count, rccl_toolset_t, api::rccl>;
|
||||
|
||||
static bool is_initialized = false;
|
||||
if(!is_initialized)
|
||||
{
|
||||
// generate the gotcha wrappers
|
||||
rcclp_gotcha_t::get_initializer() = []() {
|
||||
// TIMEMORY_C_GOTCHA(rcclp_gotcha_t, 0, ncclGetVersion);
|
||||
// TIMEMORY_C_GOTCHA(rcclp_gotcha_t, 1, ncclGetUniqueId);
|
||||
TIMEMORY_C_GOTCHA(rcclp_gotcha_t, 2, ncclCommInitRank);
|
||||
TIMEMORY_C_GOTCHA(rcclp_gotcha_t, 3, ncclCommInitAll);
|
||||
TIMEMORY_C_GOTCHA(rcclp_gotcha_t, 4, ncclCommDestroy);
|
||||
TIMEMORY_C_GOTCHA(rcclp_gotcha_t, 5, ncclCommCount);
|
||||
TIMEMORY_C_GOTCHA(rcclp_gotcha_t, 6, ncclCommCuDevice);
|
||||
TIMEMORY_C_GOTCHA(rcclp_gotcha_t, 7, ncclCommUserRank);
|
||||
TIMEMORY_C_GOTCHA(rcclp_gotcha_t, 8, ncclReduce);
|
||||
TIMEMORY_C_GOTCHA(rcclp_gotcha_t, 9, ncclBcast);
|
||||
TIMEMORY_C_GOTCHA(rcclp_gotcha_t, 10, ncclBroadcast);
|
||||
TIMEMORY_C_GOTCHA(rcclp_gotcha_t, 11, ncclAllReduce);
|
||||
TIMEMORY_C_GOTCHA(rcclp_gotcha_t, 12, ncclReduceScatter);
|
||||
TIMEMORY_C_GOTCHA(rcclp_gotcha_t, 13, ncclAllGather);
|
||||
TIMEMORY_C_GOTCHA(rcclp_gotcha_t, 14, ncclGroupStart);
|
||||
TIMEMORY_C_GOTCHA(rcclp_gotcha_t, 15, ncclGroupEnd);
|
||||
TIMEMORY_C_GOTCHA(rcclp_gotcha_t, 16, ncclSend);
|
||||
TIMEMORY_C_GOTCHA(rcclp_gotcha_t, 17, ncclRecv);
|
||||
TIMEMORY_C_GOTCHA(rcclp_gotcha_t, 18, ncclGather);
|
||||
TIMEMORY_C_GOTCHA(rcclp_gotcha_t, 19, ncclScatter);
|
||||
TIMEMORY_C_GOTCHA(rcclp_gotcha_t, 20, ncclAllToAll);
|
||||
TIMEMORY_C_GOTCHA(rcclp_gotcha_t, 21, ncclAllToAllv);
|
||||
// TIMEMORY_C_GOTCHA(rcclp_gotcha_t, 22, ncclRedOpCreatePreMulSum);
|
||||
// TIMEMORY_C_GOTCHA(rcclp_gotcha_t, 23, ncclRedOpDestroy);
|
||||
};
|
||||
|
||||
// provide environment variable for suppressing wrappers
|
||||
rcclp_gotcha_t::get_reject_list() = [reject]() {
|
||||
auto _reject = reject;
|
||||
// check environment
|
||||
auto reject_list =
|
||||
tim::get_env<std::string>("OMNITRACE_RCCLP_REJECT_LIST", "");
|
||||
// add environment setting
|
||||
for(const auto& itr : tim::delimit(reject_list))
|
||||
_reject.insert(itr);
|
||||
return _reject;
|
||||
};
|
||||
|
||||
// provide environment variable for selecting wrappers
|
||||
rcclp_gotcha_t::get_permit_list() = [permit]() {
|
||||
auto _permit = permit;
|
||||
// check environment
|
||||
auto permit_list =
|
||||
tim::get_env<std::string>("OMNITRACE_RCCLP_PERMIT_LIST", "");
|
||||
// add environment setting
|
||||
for(const auto& itr : tim::delimit(permit_list))
|
||||
_permit.insert(itr);
|
||||
return _permit;
|
||||
};
|
||||
|
||||
is_initialized = true;
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
rcclp_handle::start()
|
||||
{
|
||||
if(get_tool_count()++ == 0)
|
||||
{
|
||||
get_tool_instance() = std::make_shared<rcclp_tuple_t>("timemory_rcclp");
|
||||
get_tool_instance()->start();
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
rcclp_handle::stop()
|
||||
{
|
||||
auto idx = --get_tool_count();
|
||||
if(get_tool_instance().get())
|
||||
{
|
||||
get_tool_instance()->stop();
|
||||
if(idx == 0) get_tool_instance().reset();
|
||||
}
|
||||
}
|
||||
|
||||
rcclp_handle::persistent_data&
|
||||
rcclp_handle::get_persistent_data()
|
||||
{
|
||||
static persistent_data _instance;
|
||||
return _instance;
|
||||
}
|
||||
|
||||
std::atomic<short>&
|
||||
rcclp_handle::get_configured()
|
||||
{
|
||||
return get_persistent_data().m_configured;
|
||||
}
|
||||
|
||||
rcclp_handle::toolset_ptr_t&
|
||||
rcclp_handle::get_tool_instance()
|
||||
{
|
||||
return get_persistent_data().m_tool;
|
||||
}
|
||||
|
||||
std::atomic<int64_t>&
|
||||
rcclp_handle::get_tool_count()
|
||||
{
|
||||
return get_persistent_data().m_count;
|
||||
}
|
||||
|
||||
void
|
||||
rccl_comm_data::preinit()
|
||||
{
|
||||
omnitrace::rcclp::configure();
|
||||
}
|
||||
|
||||
// ncclReduce
|
||||
void
|
||||
rccl_comm_data::audit(const gotcha_data& _data, audit::incoming, const void*, void*,
|
||||
size_t count, ncclDataType_t datatype, ncclRedOp_t, int root,
|
||||
ncclComm_t, hipStream_t)
|
||||
{
|
||||
int size = rccl_type_size(datatype);
|
||||
add(_data, count * size, JOIN('_', _data.tool_id.c_str(), "root", root));
|
||||
}
|
||||
|
||||
// ncclSend
|
||||
void
|
||||
rccl_comm_data::audit(const gotcha_data& _data, audit::incoming, const void*,
|
||||
size_t count, ncclDataType_t datatype, int peer, ncclComm_t,
|
||||
hipStream_t)
|
||||
{
|
||||
int size = rccl_type_size(datatype);
|
||||
add(_data, count * size, JOIN('_', _data.tool_id.c_str(), "root", peer));
|
||||
}
|
||||
|
||||
// ncclBcast
|
||||
// ncclRecv
|
||||
void
|
||||
rccl_comm_data::audit(const gotcha_data& _data, audit::incoming, void*, size_t count,
|
||||
ncclDataType_t datatype, int root, ncclComm_t, hipStream_t)
|
||||
{
|
||||
int size = rccl_type_size(datatype);
|
||||
add(_data, count * size, JOIN('_', _data.tool_id.c_str(), "root", root));
|
||||
}
|
||||
|
||||
// ncclBroadcast
|
||||
void
|
||||
rccl_comm_data::audit(const gotcha_data& _data, audit::incoming, const void*, void*,
|
||||
size_t count, ncclDataType_t datatype, int root, ncclComm_t,
|
||||
hipStream_t)
|
||||
{
|
||||
int size = rccl_type_size(datatype);
|
||||
add(_data, count * size, JOIN('_', _data.tool_id.c_str(), "root", root));
|
||||
}
|
||||
|
||||
// ncclAllReduce
|
||||
// ncclReduceScatter
|
||||
void
|
||||
rccl_comm_data::audit(const gotcha_data& _data, audit::incoming, const void*, void*,
|
||||
size_t count, ncclDataType_t datatype, ncclRedOp_t, ncclComm_t,
|
||||
hipStream_t)
|
||||
{
|
||||
int size = rccl_type_size(datatype);
|
||||
add(_data, count * size);
|
||||
}
|
||||
|
||||
// ncclAllGather
|
||||
void
|
||||
rccl_comm_data::audit(const gotcha_data& _data, audit::incoming, const void*, void*,
|
||||
size_t count, ncclDataType_t datatype, ncclComm_t, hipStream_t)
|
||||
{
|
||||
int size = rccl_type_size(datatype);
|
||||
add(_data, count * size);
|
||||
}
|
||||
|
||||
} // namespace component
|
||||
} // namespace tim
|
||||
|
||||
TIMEMORY_INITIALIZE_STORAGE(rccl_comm_data, rccl_data_tracker_t)
|
||||
Reference in New Issue
Block a user