[rocprofiler-sdk][RCCL] RCCL New API changes for RCCL_API_TRACE_VERSION_PATCH = 2 (#985)
- Address build issue with RCCL sync with NCCL commit: ROCm/rccl@08a7be2 - Patch Version Bump-up PR: ROCm/rccl#1916
This commit is contained in:
Venkateshwar Reddy Kandula
zatwierdzone przez
GitHub
GitHub
rodzic
d1ee1f0cba
commit
c441a87a00
@@ -73,12 +73,18 @@ ROCP_SDK_ENFORCE_ABI(::rcclApiFuncTable, ncclCommRegister_fn, 35)
|
||||
ROCP_SDK_ENFORCE_ABI(::rcclApiFuncTable, ncclCommDeregister_fn, 36)
|
||||
#if RCCL_API_TRACE_VERSION_PATCH >= 1
|
||||
ROCP_SDK_ENFORCE_ABI(::rcclApiFuncTable, ncclAllReduceWithBias_fn, 37)
|
||||
#elif RCCL_API_TRACE_VERSION_PATCH >= 2
|
||||
ROCP_SDK_ENFORCE_ABI(::rcclApiFuncTable, ncclCommShrink_fn, 38)
|
||||
ROCP_SDK_ENFORCE_ABI(::rcclApiFuncTable, ncclCommWindowRegister_fn, 39)
|
||||
ROCP_SDK_ENFORCE_ABI(::rcclApiFuncTable, ncclCommWindowDeregister_fn, 40)
|
||||
#endif
|
||||
|
||||
#if RCCL_API_TRACE_VERSION_PATCH == 0
|
||||
ROCP_SDK_ENFORCE_ABI_VERSIONING(::rcclApiFuncTable, 37)
|
||||
#elif RCCL_API_TRACE_VERSION_PATCH == 1
|
||||
ROCP_SDK_ENFORCE_ABI_VERSIONING(::rcclApiFuncTable, 38)
|
||||
#elif RCCL_API_TRACE_VERSION_PATCH == 2
|
||||
ROCP_SDK_ENFORCE_ABI_VERSIONING(::rcclApiFuncTable, 41)
|
||||
#else
|
||||
INTERNAL_CI_ROCP_SDK_ENFORCE_ABI_VERSIONING(::rcclApiFuncTable, 0)
|
||||
#endif
|
||||
|
||||
@@ -105,6 +105,11 @@ RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_nc
|
||||
#if RCCL_API_TRACE_VERSION_PATCH >= 1
|
||||
RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclAllReduceWithBias, ncclAllReduceWithBias, ncclAllReduceWithBias_fn, sendbuff, recvbuff, count, datatype, op, comm, stream, acc)
|
||||
#endif
|
||||
#if RCCL_API_TRACE_VERSION_PATCH >= 2
|
||||
RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclCommShrink, ncclCommShrink, ncclCommShrink_fn, comm, excludeRanksList, excludeRanksCount, newcomm, config, shrinkFlags)
|
||||
RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclCommWindowRegister, ncclCommWindowRegister, ncclCommWindowRegister_fn, comm, buff, size, win, winFlags)
|
||||
RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclCommWindowDeregister, ncclCommWindowDeregister, ncclCommWindowDeregister_fn, comm, win)
|
||||
#endif
|
||||
|
||||
#else
|
||||
# error \
|
||||
|
||||
@@ -1160,21 +1160,72 @@ rocprofiler_set_api_table(const char* name,
|
||||
|
||||
auto* rccl_api = static_cast<rcclApiFuncTable*>(tables[0]);
|
||||
|
||||
// any internal modifications to the rcclApiFuncTable need to be done before we make the
|
||||
// copy or else those modifications will be lost when RCCL API tracing is enabled
|
||||
// because the RCCL API tracing invokes the function pointers from the copy below
|
||||
rocprofiler::rccl::copy_table(rccl_api, lib_instance);
|
||||
auto is_valid_rccl_dispatch_table = (rccl_api != nullptr);
|
||||
|
||||
// install rocprofiler API wrappers
|
||||
rocprofiler::rccl::update_table(rccl_api);
|
||||
// Runtime ABI validation for RCCL API dispatch table.
|
||||
//
|
||||
// NOTE: These checks are necessary because rocprofiler-sdk enforces ABI
|
||||
// compatibility at compile time. If RCCL is rebuilt afterwards with an
|
||||
// incorrect or mismatched dispatch table, compile-time checks are bypassed.
|
||||
#if ROCPROFILER_SDK_COMPUTE_VERSION(RCCL_API_TRACE_VERSION_MAJOR, \
|
||||
0, \
|
||||
RCCL_API_TRACE_VERSION_PATCH) >= 1
|
||||
// 1. For RCCL_API_TRACE_VERSION_PATCH = 1, ncclAllReduceWithBias_fn is expected
|
||||
// to be the last entry (38th function) in the dispatch table. Its offset is
|
||||
// therefore used as the canonical end of the table for patch 1.
|
||||
//
|
||||
// Problem: Some intermediate RCCL commits introduced new APIs *before*
|
||||
// ncclAllReduceWithBias_fn without bumping the ABI patch version. That
|
||||
// breaks the ABI contract with rocprofiler-sdk, because the table layout no
|
||||
// longer matches what the SDK was compiled against.
|
||||
//
|
||||
// 2. This check prevents such mismatches at runtime:
|
||||
// a. NCCL_VERSION_CODE < 22703 → indicates the first RCCL build was taken
|
||||
// before the broken commits.
|
||||
// b. rccl_api->size > offsetof(..., ncclAllReduceWithBias_fn) + sizeof(void*)
|
||||
// → indicates the current RCCL dispatch table is larger than expected,
|
||||
// meaning newer (broken) entries were inserted before the known last API.
|
||||
//
|
||||
// If both conditions are true, the dispatch table is invalid and tracing is
|
||||
// disabled to avoid corrupt output.
|
||||
if(is_valid_rccl_dispatch_table && NCCL_VERSION_CODE < 22703 &&
|
||||
rccl_api->size > offsetof(rcclApiFuncTable, ncclAllReduceWithBias_fn) + sizeof(void*))
|
||||
{
|
||||
is_valid_rccl_dispatch_table = false;
|
||||
|
||||
// Tracing notifications the runtime has initialized
|
||||
rocprofiler::runtime_init::initialize(
|
||||
ROCPROFILER_RUNTIME_INITIALIZATION_RCCL, lib_version, lib_instance);
|
||||
ROCP_CI_LOG(WARNING) << fmt::format(
|
||||
"Invalid RCCL dispatch table: layout does not match the expected "
|
||||
"rocprofiler-SDK ABI (RCCL API Trace v{}.{}.{}). "
|
||||
"Tracing is disabled to prevent corrupted data. "
|
||||
"Use a compatible RCCL version.",
|
||||
RCCL_API_TRACE_VERSION_MAJOR,
|
||||
0,
|
||||
RCCL_API_TRACE_VERSION_PATCH);
|
||||
}
|
||||
#endif
|
||||
if(is_valid_rccl_dispatch_table)
|
||||
{
|
||||
// any internal modifications to the rcclApiFuncTable need to be done before we make
|
||||
// the copy or else those modifications will be lost when RCCL API tracing is
|
||||
// enabled because the RCCL API tracing invokes the function pointers from the copy
|
||||
// below
|
||||
rocprofiler::rccl::copy_table(rccl_api, lib_instance);
|
||||
|
||||
// allow tools to install API wrappers
|
||||
rocprofiler::intercept_table::notify_intercept_table_registration(
|
||||
ROCPROFILER_RCCL_TABLE, lib_version, lib_instance, std::make_tuple(rccl_api));
|
||||
// install rocprofiler API wrappers
|
||||
rocprofiler::rccl::update_table(rccl_api);
|
||||
|
||||
// Tracing notifications the runtime has initialized
|
||||
rocprofiler::runtime_init::initialize(
|
||||
ROCPROFILER_RUNTIME_INITIALIZATION_RCCL, lib_version, lib_instance);
|
||||
|
||||
// allow tools to install API wrappers
|
||||
rocprofiler::intercept_table::notify_intercept_table_registration(
|
||||
ROCPROFILER_RCCL_TABLE, lib_version, lib_instance, std::make_tuple(rccl_api));
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCP_CI_LOG(WARNING) << "RCCL API tracing is disabled: dispatch table is invalid.";
|
||||
}
|
||||
}
|
||||
else if(std::string_view{name} == "rocdecode")
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user