diff --git a/cmake/rocprofiler_config_interfaces.cmake b/cmake/rocprofiler_config_interfaces.cmake index 9e99fbdabd..d0860f1478 100644 --- a/cmake/rocprofiler_config_interfaces.cmake +++ b/cmake/rocprofiler_config_interfaces.cmake @@ -295,3 +295,23 @@ target_link_libraries(rocprofiler-elfio INTERFACE elfio::elfio) # ----------------------------------------------------------------------------------------# target_link_libraries(rocprofiler-otf2 INTERFACE otf2::otf2) + +# ----------------------------------------------------------------------------------------# +# +# RCCL +# +# ----------------------------------------------------------------------------------------# +find_package( + rccl + REQUIRED + CONFIG + HINTS + ${rocm_version_DIR} + ${ROCM_PATH} + PATHS + ${rocm_version_DIR} + ${ROCM_PATH} + PATH_SUFFIXES + lib/cmake/rccl) + +rocprofiler_config_nolink_target(rocprofiler-rccl-nolink rccl::rccl) diff --git a/cmake/rocprofiler_interfaces.cmake b/cmake/rocprofiler_interfaces.cmake index 01ca370c37..c07b4881b1 100644 --- a/cmake/rocprofiler_interfaces.cmake +++ b/cmake/rocprofiler_interfaces.cmake @@ -82,3 +82,5 @@ rocprofiler_add_interface_library( rocprofiler_add_interface_library( rocprofiler-hsakmt-nolink "rocprofiler-hsakmt without linking to HSAKMT library" IMPORTED) +rocprofiler_add_interface_library(rocprofiler-rccl-nolink + "RCCL headers without linking to RCCL library" IMPORTED) diff --git a/source/bin/rocprofv3.py b/source/bin/rocprofv3.py index d4aeeb2241..bc0146b243 100755 --- a/source/bin/rocprofv3.py +++ b/source/bin/rocprofv3.py @@ -141,13 +141,13 @@ For MPI applications (or other job launchers such as SLURM), place rocprofv3 ins aggregate_tracing_options, "-r", "--runtime-trace", - help="Collect tracing data for HIP runtime API, Marker (ROCTx) API, Memory operations (copies and scratch), and Kernel dispatches. Similar to --sys-trace but without tracing HIP compiler API and the underlying HSA API.", + help="Collect tracing data for HIP runtime API, Marker (ROCTx) API, RCCL API, Memory operations (copies and scratch), and Kernel dispatches. Similar to --sys-trace but without tracing HIP compiler API and the underlying HSA API.", ) add_parser_bool_argument( aggregate_tracing_options, "-s", "--sys-trace", - help="Collect tracing data for HIP API, HSA API, Marker (ROCTx) API, Memory operations (copies and scratch), and Kernel dispatches.", + help="Collect tracing data for HIP API, HSA API, Marker (ROCTx) API, RCCL API, Memory operations (copies and scratch), and Kernel dispatches.", ) basic_tracing_options = parser.add_argument_group("Basic tracing options") @@ -183,6 +183,11 @@ For MPI applications (or other job launchers such as SLURM), place rocprofv3 ins "--hsa-trace", help="For collecting HSA Traces (core + amd + image + finalizer)", ) + add_parser_bool_argument( + basic_tracing_options, + "--rccl-trace", + help="For collecting RCCL Traces", + ) extended_tracing_options = parser.add_argument_group("Granular tracing options") @@ -654,6 +659,7 @@ def run(app_args, args, **kwargs): "kernel_trace", "memory_copy_trace", "scratch_memory_trace", + "rccl_trace", ): setattr(args, itr, True) @@ -664,6 +670,7 @@ def run(app_args, args, **kwargs): "kernel_trace", "memory_copy_trace", "scratch_memory_trace", + "rccl_trace", ): setattr(args, itr, True) @@ -686,6 +693,7 @@ def run(app_args, args, **kwargs): ["hsa_image_trace", "HSA_IMAGE_EXT_API_TRACE"], ["hsa_finalizer_trace", "HSA_FINALIZER_EXT_API_TRACE"], ["marker_trace", "MARKER_API_TRACE"], + ["rccl_trace", "RCCL_API_TRACE"], ["kernel_trace", "KERNEL_TRACE"], ["memory_copy_trace", "MEMORY_COPY_TRACE"], ["scratch_memory_trace", "SCRATCH_MEMORY_TRACE"], diff --git a/source/include/rocprofiler-sdk/buffer_tracing.h b/source/include/rocprofiler-sdk/buffer_tracing.h index 505095540d..12de3fb342 100644 --- a/source/include/rocprofiler-sdk/buffer_tracing.h +++ b/source/include/rocprofiler-sdk/buffer_tracing.h @@ -147,6 +147,24 @@ typedef struct /// ::rocprofiler_marker_name_api_id_t } rocprofiler_buffer_tracing_marker_api_record_t; +/** + * @brief ROCProfiler Buffer RCCL API Record. + */ +typedef struct +{ + uint64_t size; ///< size of this struct + rocprofiler_buffer_tracing_kind_t kind; + rocprofiler_tracing_operation_t operation; + rocprofiler_correlation_id_t correlation_id; ///< correlation ids for record + rocprofiler_timestamp_t start_timestamp; ///< start time in nanoseconds + rocprofiler_timestamp_t end_timestamp; ///< end time in nanoseconds + rocprofiler_thread_id_t thread_id; ///< id for thread generating this record + + /// @var kind + /// @brief ::ROCPROFILER_CALLBACK_TRACING_RCCL_API, + /// @brief Specification of the API function, e.g., ::rocprofiler_rccl_api_id_t, +} rocprofiler_buffer_tracing_rccl_api_record_t; + /** * @brief ROCProfiler Buffer Memory Copy Tracer Record. */ diff --git a/source/include/rocprofiler-sdk/callback_tracing.h b/source/include/rocprofiler-sdk/callback_tracing.h index 6f50d8aa50..124207d402 100644 --- a/source/include/rocprofiler-sdk/callback_tracing.h +++ b/source/include/rocprofiler-sdk/callback_tracing.h @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -87,6 +88,16 @@ typedef struct rocprofiler_marker_api_retval_t retval; } rocprofiler_callback_tracing_marker_api_data_t; +/** + * @brief ROCProfiler RCCL API Callback Data. + */ +typedef struct +{ + uint64_t size; ///< size of this struct + rocprofiler_rccl_api_args_t args; + rocprofiler_rccl_api_retval_t retval; +} rocprofiler_callback_tracing_rccl_api_data_t; + /** * @brief ROCProfiler Code Object Load Tracer Callback Record. */ diff --git a/source/include/rocprofiler-sdk/cxx/perfetto.hpp b/source/include/rocprofiler-sdk/cxx/perfetto.hpp index 6ceb5165b6..2e39ef7daf 100644 --- a/source/include/rocprofiler-sdk/cxx/perfetto.hpp +++ b/source/include/rocprofiler-sdk/cxx/perfetto.hpp @@ -77,6 +77,7 @@ struct perfetto_category; ROCPROFILER_DEFINE_CATEGORY(category, hsa_api, "HSA API function") ROCPROFILER_DEFINE_CATEGORY(category, hip_api, "HIP API function") ROCPROFILER_DEFINE_CATEGORY(category, marker_api, "Marker API region") +ROCPROFILER_DEFINE_CATEGORY(category, rccl_api, "RCCL API function") ROCPROFILER_DEFINE_CATEGORY(category, kernel_dispatch, "GPU kernel dispatch") ROCPROFILER_DEFINE_CATEGORY(category, memory_copy, "Async memory copy") @@ -84,6 +85,7 @@ ROCPROFILER_DEFINE_CATEGORY(category, memory_copy, "Async memory copy") ROCPROFILER_PERFETTO_CATEGORY(category::hsa_api), \ ROCPROFILER_PERFETTO_CATEGORY(category::hip_api), \ ROCPROFILER_PERFETTO_CATEGORY(category::marker_api), \ + ROCPROFILER_PERFETTO_CATEGORY(category::rccl_api), \ ROCPROFILER_PERFETTO_CATEGORY(category::kernel_dispatch), \ ROCPROFILER_PERFETTO_CATEGORY(category::memory_copy) diff --git a/source/include/rocprofiler-sdk/cxx/serialization.hpp b/source/include/rocprofiler-sdk/cxx/serialization.hpp index 3b2b53ca9d..e9f278076d 100644 --- a/source/include/rocprofiler-sdk/cxx/serialization.hpp +++ b/source/include/rocprofiler-sdk/cxx/serialization.hpp @@ -327,6 +327,22 @@ save(ArchiveT& ar, rocprofiler_callback_tracing_memory_copy_data_t data) ROCP_SDK_SAVE_DATA_FIELD(bytes); } +template +void +save(ArchiveT& ar, rocprofiler_rccl_api_retval_t data) +{ + ROCP_SDK_SAVE_DATA_FIELD(ncclResult_t_retval); +} + +template +void +save(ArchiveT& ar, rocprofiler_callback_tracing_rccl_api_data_t data) +{ + ROCP_SDK_SAVE_DATA_FIELD(size); + // ROCP_SDK_SAVE_DATA_FIELD(args); + ROCP_SDK_SAVE_DATA_FIELD(retval); +} + template void save(ArchiveT& ar, rocprofiler_profile_counting_dispatch_data_t data) @@ -405,6 +421,13 @@ save(ArchiveT& ar, rocprofiler_buffer_tracing_marker_api_record_t data) save_buffer_tracing_api_record(ar, data); } +template +void +save(ArchiveT& ar, rocprofiler_buffer_tracing_rccl_api_record_t data) +{ + save_buffer_tracing_api_record(ar, data); +} + template void save(ArchiveT& ar, rocprofiler_buffer_tracing_kernel_dispatch_record_t data) diff --git a/source/include/rocprofiler-sdk/external_correlation.h b/source/include/rocprofiler-sdk/external_correlation.h index 4155736cba..3a7f9e8dad 100644 --- a/source/include/rocprofiler-sdk/external_correlation.h +++ b/source/include/rocprofiler-sdk/external_correlation.h @@ -66,6 +66,7 @@ typedef enum // NOLINT(performance-enum-size) ROCPROFILER_EXTERNAL_CORRELATION_REQUEST_MEMORY_COPY, ROCPROFILER_EXTERNAL_CORRELATION_REQUEST_KERNEL_DISPATCH, ROCPROFILER_EXTERNAL_CORRELATION_REQUEST_SCRATCH_MEMORY, + ROCPROFILER_EXTERNAL_CORRELATION_REQUEST_RCCL_API, ROCPROFILER_EXTERNAL_CORRELATION_REQUEST_LAST, } rocprofiler_external_correlation_id_request_kind_t; diff --git a/source/include/rocprofiler-sdk/fwd.h b/source/include/rocprofiler-sdk/fwd.h index f70b6a4262..9075f654db 100644 --- a/source/include/rocprofiler-sdk/fwd.h +++ b/source/include/rocprofiler-sdk/fwd.h @@ -169,6 +169,7 @@ typedef enum // NOLINT(performance-enum-size) ROCPROFILER_CALLBACK_TRACING_SCRATCH_MEMORY, ///< @see ::rocprofiler_scratch_memory_operation_t ROCPROFILER_CALLBACK_TRACING_KERNEL_DISPATCH, ///< Callbacks for kernel dispatches ROCPROFILER_CALLBACK_TRACING_MEMORY_COPY, ///< @see ::rocprofiler_memory_copy_operation_t + ROCPROFILER_CALLBACK_TRACING_RCCL_API, ///< @RCCL tracing ROCPROFILER_CALLBACK_TRACING_LAST, } rocprofiler_callback_tracing_kind_t; @@ -193,6 +194,7 @@ typedef enum // NOLINT(performance-enum-size) ROCPROFILER_BUFFER_TRACING_PAGE_MIGRATION, ///< Buffer page migration info ROCPROFILER_BUFFER_TRACING_SCRATCH_MEMORY, ///< Buffer scratch memory reclaimation info ROCPROFILER_BUFFER_TRACING_CORRELATION_ID_RETIREMENT, ///< Correlation ID in no longer in use + ROCPROFILER_BUFFER_TRACING_RCCL_API, ///< RCCL tracing ROCPROFILER_BUFFER_TRACING_LAST, } rocprofiler_buffer_tracing_kind_t; @@ -350,7 +352,8 @@ typedef enum ROCPROFILER_HSA_LIBRARY = (1 << 1), ROCPROFILER_HIP_LIBRARY = (1 << 2), ROCPROFILER_MARKER_LIBRARY = (1 << 3), - ROCPROFILER_LIBRARY_LAST = ROCPROFILER_MARKER_LIBRARY, + ROCPROFILER_RCCL_LIBRARY = (1 << 4), + ROCPROFILER_LIBRARY_LAST = ROCPROFILER_RCCL_LIBRARY, } rocprofiler_runtime_library_t; /** @@ -365,7 +368,8 @@ typedef enum ROCPROFILER_MARKER_CORE_TABLE = (1 << 3), ROCPROFILER_MARKER_CONTROL_TABLE = (1 << 4), ROCPROFILER_MARKER_NAME_TABLE = (1 << 5), - ROCPROFILER_TABLE_LAST = ROCPROFILER_MARKER_NAME_TABLE, + ROCPROFILER_RCCL_TABLE = (1 << 6), + ROCPROFILER_TABLE_LAST = ROCPROFILER_RCCL_TABLE, } rocprofiler_intercept_table_t; /** diff --git a/source/include/rocprofiler-sdk/rccl.h b/source/include/rocprofiler-sdk/rccl.h new file mode 100644 index 0000000000..008fd82fd2 --- /dev/null +++ b/source/include/rocprofiler-sdk/rccl.h @@ -0,0 +1,27 @@ +// MIT License +// +// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#pragma once + +#include +#include +#include diff --git a/source/include/rocprofiler-sdk/rccl/api_args.h b/source/include/rocprofiler-sdk/rccl/api_args.h new file mode 100644 index 0000000000..651303dfdf --- /dev/null +++ b/source/include/rocprofiler-sdk/rccl/api_args.h @@ -0,0 +1,308 @@ +// MIT License +// +// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#pragma once + +#include +#include + +#include + +#include + +ROCPROFILER_EXTERN_C_INIT + +// Empty struct has a size of 0 in C but size of 1 in C++. +// This struct is added to the union members which represent +// functions with no arguments to ensure ABI compatibility +typedef struct rocprofiler_rccl_api_no_args +{ + char empty; +} rocprofiler_rccl_api_no_args; + +typedef union rocprofiler_rccl_api_retval_t +{ + int32_t ncclResult_t_retval; + const char* const_charp_retval; +} rocprofiler_rccl_api_retval_t; + +typedef union rocprofiler_rccl_api_args_t +{ + struct + { + const void* sendbuff; + void* recvbuff; + size_t sendcount; + ncclDataType_t datatype; + ncclComm_t comm; + hipStream_t stream; + } ncclAllGather; + struct + { + const void* sendbuff; + void* recvbuff; + size_t count; + ncclDataType_t datatype; + ncclRedOp_t op; + struct ncclComm* comm; + hipStream_t stream; + } ncclAllReduce; + struct + { + const void* sendbuff; + void* recvbuff; + size_t count; + ncclDataType_t datatype; + ncclComm_t comm; + hipStream_t stream; + } ncclAllToAll; + struct + { + const void* sendbuff; + const size_t* sendcounts; + const size_t* sdispls; + void* recvbuff; + const size_t* recvcounts; + const size_t* rdispls; + ncclDataType_t datatype; + ncclComm_t comm; + hipStream_t stream; + } ncclAllToAllv; + struct + { + const void* sendbuff; + void* recvbuff; + size_t count; + ncclDataType_t datatype; + int root; + ncclComm_t comm; + hipStream_t stream; + } ncclBroadcast; + struct + { + const void* sendbuff; + void* recvbuff; + size_t sendcount; + ncclDataType_t datatype; + int root; + ncclComm_t comm; + hipStream_t stream; + } ncclGather; + struct + { + const void* sendbuff; + void* recvbuff; + size_t count; + ncclDataType_t datatype; + ncclRedOp_t op; + int root; + ncclComm_t comm; + hipStream_t stream; + } ncclReduce; + struct + { + const void* sendbuff; + void* recvbuff; + size_t recvcount; + ncclDataType_t datatype; + ncclRedOp_t op; + struct ncclComm* comm; + hipStream_t stream; + } ncclReduceScatter; + struct + { + const void* sendbuff; + void* recvbuff; + size_t recvcount; + ncclDataType_t datatype; + int root; + ncclComm_t comm; + hipStream_t stream; + } ncclScatter; + struct + { + const void* sendbuff; + size_t count; + ncclDataType_t datatype; + int peer; + ncclComm_t comm; + hipStream_t stream; + } ncclSend; + struct + { + void* recvbuff; + size_t count; + ncclDataType_t datatype; + int peer; + ncclComm_t comm; + hipStream_t stream; + } ncclRecv; + struct + { + ncclRedOp_t* op; + void* scalar; + ncclDataType_t datatype; + ncclScalarResidence_t residence; + ncclComm_t comm; + } ncclRedOpCreatePreMulSum; + struct + { + ncclRedOp_t op; + ncclComm_t comm; + } ncclRedOpDestroy; + struct + { + rocprofiler_rccl_api_no_args no_args; + } ncclGroupStart; + struct + { + rocprofiler_rccl_api_no_args no_args; + } ncclGroupEnd; + struct + { + int* version; + } ncclGetVersion; + struct + { + ncclUniqueId* out; + } ncclGetUniqueId; + struct + { + ncclComm_t* newcomm; + int nranks; + ncclUniqueId commId; + int myrank; + } ncclCommInitRank; + struct + { + ncclComm_t* comms; + int ndev; + const int* devlist; + } ncclCommInitAll; + struct + { + ncclComm_t* comm; + int nranks; + ncclUniqueId commId; + int myrank; + ncclConfig_t* config; + } ncclCommInitRankConfig; + struct + { + ncclComm_t comm; + } ncclCommFinalize; + struct + { + ncclComm_t comm; + } ncclCommDestroy; + struct + { + ncclComm_t comm; + } ncclCommAbort; + struct + { + ncclComm_t comm; + int color; + int key; + ncclComm_t* newcomm; + ncclConfig_t* config; + } ncclCommSplit; + struct + { + ncclResult_t code; + } ncclGetErrorString; + struct + { + ncclComm_t comm; + } ncclGetLastError; + struct + { + ncclComm_t comm; + ncclResult_t* asyncError; + } ncclCommGetAsyncError; + struct + { + ncclComm_t comm; + int* count; + } ncclCommCount; + struct + { + ncclComm_t comm; + int* devid; + } ncclCommCuDevice; + struct + { + ncclComm_t comm; + int* rank; + } ncclCommUserRank; + struct + { + void** ptr; + size_t size; + } ncclMemAlloc; + struct + { + void* ptr; + } ncclMemFree; + struct + { + const char* mscclAlgoFilePath; + mscclAlgoHandle_t* mscclAlgoHandle; + int rank; + } mscclLoadAlgo; + struct + { + const void* sendBuff; + const size_t* sendCounts; + const size_t* sDisPls; + void* recvBuff; + const size_t* recvCounts; + const size_t* rDisPls; + size_t count; + ncclDataType_t dataType; + int root; + int peer; + ncclRedOp_t op; + mscclAlgoHandle_t mscclAlgoHandle; + ncclComm_t comm; + hipStream_t stream; + } mscclRunAlgo; + struct + { + mscclAlgoHandle_t mscclAlgoHandle; + } mscclUnloadAlgo; + struct + { + ncclComm_t comm; + void* buff; + size_t size; + void** handle; + } ncclCommRegister; + struct + { + ncclComm_t comm; + void* handle; + } ncclCommDeregister; + +} rocprofiler_rccl_api_args_t; + +ROCPROFILER_EXTERN_C_FINI diff --git a/source/include/rocprofiler-sdk/rccl/api_id.h b/source/include/rocprofiler-sdk/rccl/api_id.h new file mode 100644 index 0000000000..7ef17bea61 --- /dev/null +++ b/source/include/rocprofiler-sdk/rccl/api_id.h @@ -0,0 +1,75 @@ + + +// MIT License +// +// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#pragma once + +#include + +/** + * @brief ROCProfiler enumeration of HSA Core API tracing operations + */ +typedef enum // NOLINT(performance-enum-size) +{ + ROCPROFILER_RCCL_API_ID_NONE = -1, + + ROCPROFILER_RCCL_API_ID_ncclAllGather = 0, + ROCPROFILER_RCCL_API_ID_ncclAllReduce, + ROCPROFILER_RCCL_API_ID_ncclAllToAll, + ROCPROFILER_RCCL_API_ID_ncclAllToAllv, + ROCPROFILER_RCCL_API_ID_ncclBroadcast, + ROCPROFILER_RCCL_API_ID_ncclGather, + ROCPROFILER_RCCL_API_ID_ncclReduce, + ROCPROFILER_RCCL_API_ID_ncclReduceScatter, + ROCPROFILER_RCCL_API_ID_ncclScatter, + ROCPROFILER_RCCL_API_ID_ncclSend, + ROCPROFILER_RCCL_API_ID_ncclRecv, + ROCPROFILER_RCCL_API_ID_ncclRedOpCreatePreMulSum, + ROCPROFILER_RCCL_API_ID_ncclRedOpDestroy, + ROCPROFILER_RCCL_API_ID_ncclGroupStart, + ROCPROFILER_RCCL_API_ID_ncclGroupEnd, + ROCPROFILER_RCCL_API_ID_ncclGetVersion, + ROCPROFILER_RCCL_API_ID_ncclGetUniqueId, + ROCPROFILER_RCCL_API_ID_ncclCommInitRank, + ROCPROFILER_RCCL_API_ID_ncclCommInitAll, + ROCPROFILER_RCCL_API_ID_ncclCommInitRankConfig, + ROCPROFILER_RCCL_API_ID_ncclCommFinalize, + ROCPROFILER_RCCL_API_ID_ncclCommDestroy, + ROCPROFILER_RCCL_API_ID_ncclCommAbort, + ROCPROFILER_RCCL_API_ID_ncclCommSplit, + ROCPROFILER_RCCL_API_ID_ncclGetErrorString, + ROCPROFILER_RCCL_API_ID_ncclGetLastError, + ROCPROFILER_RCCL_API_ID_ncclCommGetAsyncError, + ROCPROFILER_RCCL_API_ID_ncclCommCount, + ROCPROFILER_RCCL_API_ID_ncclCommCuDevice, + ROCPROFILER_RCCL_API_ID_ncclCommUserRank, + ROCPROFILER_RCCL_API_ID_ncclMemAlloc, + ROCPROFILER_RCCL_API_ID_ncclMemFree, + ROCPROFILER_RCCL_API_ID_mscclLoadAlgo, + ROCPROFILER_RCCL_API_ID_mscclRunAlgo, + ROCPROFILER_RCCL_API_ID_mscclUnloadAlgo, + ROCPROFILER_RCCL_API_ID_ncclCommRegister, + ROCPROFILER_RCCL_API_ID_ncclCommDeregister, + + ROCPROFILER_RCCL_API_ID_LAST, +} rocprofiler_rccl_api_id_t; diff --git a/source/include/rocprofiler-sdk/rccl/table_id.h b/source/include/rocprofiler-sdk/rccl/table_id.h new file mode 100644 index 0000000000..578d3da290 --- /dev/null +++ b/source/include/rocprofiler-sdk/rccl/table_id.h @@ -0,0 +1,31 @@ +// MIT License +// +// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#pragma once + +// NOLINTNEXTLINE(performance-enum-size) +typedef enum +{ + ROCPROFILER_RCCL_TABLE_ID_NONE = -1, + ROCPROFILER_RCCL_TABLE_ID = 0, + ROCPROFILER_RCCL_TABLE_ID_LAST, +} rocprofiler_rccl_table_id_t; diff --git a/source/lib/rocprofiler-sdk-tool/config.hpp b/source/lib/rocprofiler-sdk-tool/config.hpp index 55f7b0d1bc..d4557d3239 100644 --- a/source/lib/rocprofiler-sdk-tool/config.hpp +++ b/source/lib/rocprofiler-sdk-tool/config.hpp @@ -92,6 +92,7 @@ struct config bool counter_collection = get_env("ROCPROF_COUNTER_COLLECTION", false); bool hip_runtime_api_trace = get_env("ROCPROF_HIP_RUNTIME_API_TRACE", false); bool hip_compiler_api_trace = get_env("ROCPROF_HIP_COMPILER_API_TRACE", false); + bool rccl_api_trace = get_env("ROCPROF_RCCL_API_TRACE", false); bool list_metrics = get_env("ROCPROF_LIST_METRICS", false); bool list_metrics_output_file = get_env("ROCPROF_OUTPUT_LIST_METRICS_FILE", false); bool stats = get_env("ROCPROF_STATS", false); diff --git a/source/lib/rocprofiler-sdk-tool/domain_type.cpp b/source/lib/rocprofiler-sdk-tool/domain_type.cpp index d8cacc6f4f..6686b6936a 100644 --- a/source/lib/rocprofiler-sdk-tool/domain_type.cpp +++ b/source/lib/rocprofiler-sdk-tool/domain_type.cpp @@ -44,6 +44,7 @@ DEFINE_BUFFER_TYPE_NAME(KERNEL_DISPATCH, "KERNEL_DISPATCH", "kernel_dispatch") DEFINE_BUFFER_TYPE_NAME(MEMORY_COPY, "MEMORY_COPY", "memory_copy") DEFINE_BUFFER_TYPE_NAME(SCRATCH_MEMORY, "SCRATCH_MEMORY", "scratch_memory") DEFINE_BUFFER_TYPE_NAME(COUNTER_COLLECTION, "COUNTER_COLLECTION", "counter_collection") +DEFINE_BUFFER_TYPE_NAME(RCCL, "RCCL_API", "rccl_api") #undef DEFINE_BUFFER_TYPE_NAME diff --git a/source/lib/rocprofiler-sdk-tool/domain_type.hpp b/source/lib/rocprofiler-sdk-tool/domain_type.hpp index e4b055b820..15bf4c0dcb 100644 --- a/source/lib/rocprofiler-sdk-tool/domain_type.hpp +++ b/source/lib/rocprofiler-sdk-tool/domain_type.hpp @@ -33,6 +33,7 @@ enum class domain_type MEMORY_COPY, SCRATCH_MEMORY, COUNTER_COLLECTION, + RCCL, LAST, }; diff --git a/source/lib/rocprofiler-sdk-tool/generateCSV.cpp b/source/lib/rocprofiler-sdk-tool/generateCSV.cpp index 8841a19136..3d91c817cd 100644 --- a/source/lib/rocprofiler-sdk-tool/generateCSV.cpp +++ b/source/lib/rocprofiler-sdk-tool/generateCSV.cpp @@ -575,6 +575,43 @@ generate_csv(tool_table* } } +void +generate_csv(tool_table* tool_functions, + const std::deque& data, + const stats_entry_t& stats) +{ + if(data.empty()) return; + + if(tool::get_config().stats && stats) + write_stats(get_stats_output_file("rccl_api_stats"), stats.entries); + + auto ofs = tool::output_file{"rccl_api_trace", + tool::csv::api_csv_encoder{}, + {"Domain", + "Function", + "Process_Id", + "Thread_Id", + "Correlation_Id", + "Start_Timestamp", + "End_Timestamp"}}; + for(const auto& record : data) + { + auto row_ss = std::stringstream{}; + auto api_name = tool_functions->tool_get_operation_name_fn(record.kind, record.operation); + rocprofiler::tool::csv::api_csv_encoder::write_row( + row_ss, + tool_functions->tool_get_domain_name_fn(record.kind), + api_name, + getpid(), + record.thread_id, + record.correlation_id.internal, + record.start_timestamp, + record.end_timestamp); + + ofs << row_ss.str(); + } +} + void generate_csv(tool_table* /*tool_functions*/, const domain_stats_vec_t& data_v) { diff --git a/source/lib/rocprofiler-sdk-tool/generateCSV.hpp b/source/lib/rocprofiler-sdk-tool/generateCSV.hpp index c4cb4fefc2..743ed0446b 100644 --- a/source/lib/rocprofiler-sdk-tool/generateCSV.hpp +++ b/source/lib/rocprofiler-sdk-tool/generateCSV.hpp @@ -24,6 +24,7 @@ #include "generateStats.hpp" #include "helper.hpp" +#include "rocprofiler-sdk/buffer_tracing.h" #include "statistics.hpp" #include @@ -70,6 +71,11 @@ generate_csv(tool_table* const std::deque& data, const stats_entry_t& stats); +void +generate_csv(tool_table* tool_functions, + const std::deque& data, + const stats_entry_t& stats); + void generate_csv(tool_table* tool_functions, const domain_stats_vec_t& data); } // namespace tool diff --git a/source/lib/rocprofiler-sdk-tool/generateJSON.cpp b/source/lib/rocprofiler-sdk-tool/generateJSON.cpp index 2256d826c6..b431744741 100644 --- a/source/lib/rocprofiler-sdk-tool/generateJSON.cpp +++ b/source/lib/rocprofiler-sdk-tool/generateJSON.cpp @@ -50,7 +50,8 @@ write_json(tool_table* tool std::deque* memory_copy_deque, std::deque* counter_collection_deque, std::deque* marker_api_deque, - std::deque* scratch_memory_deque) + std::deque* scratch_memory_deque, + std::deque* rccl_api_deque) { using JSONOutputArchive = cereal::MinimalJSONOutputArchive; @@ -171,6 +172,7 @@ write_json(tool_table* tool json_ar(cereal::make_nvp("hip_api", *hip_api_deque)); json_ar(cereal::make_nvp("hsa_api", *hsa_api_deque)); json_ar(cereal::make_nvp("marker_api", *marker_api_deque)); + json_ar(cereal::make_nvp("rccl_api", *rccl_api_deque)); json_ar(cereal::make_nvp("memory_copy", *memory_copy_deque)); json_ar(cereal::make_nvp("scratch_memory", *scratch_memory_deque)); json_ar.finishNode(); diff --git a/source/lib/rocprofiler-sdk-tool/generateJSON.hpp b/source/lib/rocprofiler-sdk-tool/generateJSON.hpp index 60ea730472..2f01317677 100644 --- a/source/lib/rocprofiler-sdk-tool/generateJSON.hpp +++ b/source/lib/rocprofiler-sdk-tool/generateJSON.hpp @@ -41,7 +41,8 @@ write_json(tool_table* tool std::deque* memory_copy_deque, std::deque* counter_collection_deque, std::deque* marker_api_deque, - std::deque* scratch_memory_deque); + std::deque* scratch_memory_deque, + std::deque* rccl_api_deque); } // namespace tool } // namespace rocprofiler diff --git a/source/lib/rocprofiler-sdk-tool/generateOTF2.cpp b/source/lib/rocprofiler-sdk-tool/generateOTF2.cpp index 1e6a4e5ac3..2708af7f61 100644 --- a/source/lib/rocprofiler-sdk-tool/generateOTF2.cpp +++ b/source/lib/rocprofiler-sdk-tool/generateOTF2.cpp @@ -352,7 +352,8 @@ write_otf2(tool_table* tool std::deque* kernel_dispatch_data, std::deque* memory_copy_data, std::deque* marker_api_data, - std::deque* /*scratch_memory_data*/) + std::deque* /*scratch_memory_data*/, + std::deque* rccl_api_data) { namespace sdk = ::rocprofiler::sdk; @@ -399,6 +400,8 @@ write_otf2(tool_table* tool tids.emplace(itr.thread_id); for(auto itr : *marker_api_data) tids.emplace(itr.thread_id); + for(auto itr : *rccl_api_data) + tids.emplace(itr.thread_id); for(auto itr : *memory_copy_data) { @@ -555,6 +558,7 @@ write_otf2(tool_table* tool add_event_data(hsa_api_data, sdk::category::hsa_api{}); add_event_data(hip_api_data, sdk::category::hip_api{}); add_event_data(marker_api_data, sdk::category::marker_api{}); + add_event_data(rccl_api_data, sdk::category::rccl_api{}); } for(auto itr : *memory_copy_data) diff --git a/source/lib/rocprofiler-sdk-tool/generateOTF2.hpp b/source/lib/rocprofiler-sdk-tool/generateOTF2.hpp index a70d22099d..53c2aa1b87 100644 --- a/source/lib/rocprofiler-sdk-tool/generateOTF2.hpp +++ b/source/lib/rocprofiler-sdk-tool/generateOTF2.hpp @@ -39,6 +39,7 @@ write_otf2(tool_table* tool std::deque* kernel_dispatch_data, std::deque* memory_copy_data, std::deque* marker_api_data, - std::deque* scratch_memory_data); + std::deque* scratch_memory_data, + std::deque* rccl_api_data); } // namespace tool } // namespace rocprofiler diff --git a/source/lib/rocprofiler-sdk-tool/generatePerfetto.cpp b/source/lib/rocprofiler-sdk-tool/generatePerfetto.cpp index 6fb76c66dc..adec68f39b 100644 --- a/source/lib/rocprofiler-sdk-tool/generatePerfetto.cpp +++ b/source/lib/rocprofiler-sdk-tool/generatePerfetto.cpp @@ -71,7 +71,8 @@ write_perfetto( std::deque* kernel_dispatch_data, std::deque* memory_copy_data, std::deque* marker_api_data, - std::deque* /*scratch_memory_data*/) + std::deque* /*scratch_memory_data*/, + std::deque* rccl_api_data) { namespace sdk = ::rocprofiler::sdk; @@ -155,6 +156,8 @@ write_perfetto( tids.emplace(itr.thread_id); for(auto itr : *marker_api_data) tids.emplace(itr.thread_id); + for(auto itr : *rccl_api_data) + tids.emplace(itr.thread_id); for(auto itr : *memory_copy_data) { @@ -342,6 +345,35 @@ write_perfetto( tracing_session->FlushBlocking(); } + for(auto itr : *rccl_api_data) + { + auto name = buffer_names.at(itr.kind, itr.operation); + auto& track = thread_tracks.at(itr.thread_id); + + TRACE_EVENT_BEGIN(sdk::perfetto_category::name, + ::perfetto::StaticString(name.data()), + track, + itr.start_timestamp, + ::perfetto::Flow::ProcessScoped(itr.correlation_id.internal), + "begin_ns", + itr.start_timestamp, + "end_ns", + itr.end_timestamp, + "delta_ns", + (itr.end_timestamp - itr.start_timestamp), + "tid", + itr.thread_id, + "kind", + itr.kind, + "operation", + itr.operation, + "corr_id", + itr.correlation_id.internal); + TRACE_EVENT_END( + sdk::perfetto_category::name, track, itr.end_timestamp); + tracing_session->FlushBlocking(); + } + for(auto itr : *memory_copy_data) { auto name = buffer_names.at(itr.kind, itr.operation); diff --git a/source/lib/rocprofiler-sdk-tool/generatePerfetto.hpp b/source/lib/rocprofiler-sdk-tool/generatePerfetto.hpp index bd94ab36fc..c30ae9ce55 100644 --- a/source/lib/rocprofiler-sdk-tool/generatePerfetto.hpp +++ b/source/lib/rocprofiler-sdk-tool/generatePerfetto.hpp @@ -40,6 +40,7 @@ write_perfetto( std::deque* kernel_dispatch_data, std::deque* memory_copy_data, std::deque* marker_api_data, - std::deque* scratch_memory_data); + std::deque* scratch_memory_data, + std::deque* rccl_api_data); } // namespace tool } // namespace rocprofiler diff --git a/source/lib/rocprofiler-sdk-tool/generateStats.cpp b/source/lib/rocprofiler-sdk-tool/generateStats.cpp index 031ca17912..e03cef2bbb 100644 --- a/source/lib/rocprofiler-sdk-tool/generateStats.cpp +++ b/source/lib/rocprofiler-sdk-tool/generateStats.cpp @@ -22,16 +22,18 @@ #include "generateStats.hpp" #include "config.hpp" +#include "domain_type.hpp" #include "helper.hpp" -#include "lib/common/logging.hpp" -#include "lib/rocprofiler-sdk-tool/domain_type.hpp" #include "output_file.hpp" #include "statistics.hpp" -#include +#include "lib/common/logging.hpp" + #include #include +#include + #include #include #include @@ -166,6 +168,20 @@ generate_stats(tool_table* return get_stats(scratch_memory_stats); } +stats_entry_t +generate_stats(tool_table* tool_functions, + const std::deque& data) +{ + auto rccl_stats = stats_map_t{}; + for(const auto& record : data) + { + auto api_name = tool_functions->tool_get_operation_name_fn(record.kind, record.operation); + rccl_stats[api_name] += (record.end_timestamp - record.start_timestamp); + } + + return get_stats(rccl_stats); +} + namespace { void diff --git a/source/lib/rocprofiler-sdk-tool/generateStats.hpp b/source/lib/rocprofiler-sdk-tool/generateStats.hpp index f9124bdb59..98dd8c8154 100644 --- a/source/lib/rocprofiler-sdk-tool/generateStats.hpp +++ b/source/lib/rocprofiler-sdk-tool/generateStats.hpp @@ -57,6 +57,10 @@ stats_entry_t generate_stats(tool_table* tool_functions, const std::deque& data); +stats_entry_t +generate_stats(tool_table* tool_functions, + const std::deque& data); + void generate_stats(tool_table* tool_functions, const domain_stats_vec_t& data); } // namespace tool diff --git a/source/lib/rocprofiler-sdk-tool/helper.hpp b/source/lib/rocprofiler-sdk-tool/helper.hpp index dcdd60511d..e5d42ef998 100644 --- a/source/lib/rocprofiler-sdk-tool/helper.hpp +++ b/source/lib/rocprofiler-sdk-tool/helper.hpp @@ -324,6 +324,9 @@ using memory_copy_buffered_output_t = using marker_buffered_output_t = ::rocprofiler::tool::buffered_output; +using rccl_buffered_output_t = + ::rocprofiler::tool::buffered_output; using counter_collection_buffered_output_t = ::rocprofiler::tool::buffered_output; diff --git a/source/lib/rocprofiler-sdk-tool/tool.cpp b/source/lib/rocprofiler-sdk-tool/tool.cpp index 088f4802ee..5a2b617d67 100644 --- a/source/lib/rocprofiler-sdk-tool/tool.cpp +++ b/source/lib/rocprofiler-sdk-tool/tool.cpp @@ -44,6 +44,7 @@ #include "lib/common/utility.hpp" #include +#include #include #include #include @@ -150,15 +151,17 @@ struct buffer_ids rocprofiler_buffer_id_t memory_copy_trace = {}; rocprofiler_buffer_id_t counter_collection = {}; rocprofiler_buffer_id_t scratch_memory = {}; + rocprofiler_buffer_id_t rccl_api_trace = {}; auto as_array() const { - return std::array{hsa_api_trace, + return std::array{hsa_api_trace, hip_api_trace, kernel_trace, memory_copy_trace, counter_collection, - scratch_memory}; + scratch_memory, + rccl_api_trace}; } }; @@ -710,6 +713,13 @@ buffered_tracing_callback(rocprofiler_context_id_t /*context*/, write_ring_buffer(*record, domain_type::HIP); } + else if(header->kind == ROCPROFILER_BUFFER_TRACING_RCCL_API) + { + auto* record = + static_cast(header->payload); + + write_ring_buffer(*record, domain_type::RCCL); + } else { ROCP_FATAL << fmt::format( @@ -1423,6 +1433,28 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data) } } + if(tool::get_config().rccl_api_trace) + { + ROCPROFILER_CALL(rocprofiler_create_buffer(get_client_ctx(), + buffer_size, + buffer_watermark, + ROCPROFILER_BUFFER_POLICY_LOSSLESS, + buffered_tracing_callback, + tool_data, + &get_buffers().rccl_api_trace), + "buffer creation"); + + auto _status = + rocprofiler_configure_buffer_tracing_service(get_client_ctx(), + ROCPROFILER_BUFFER_TRACING_RCCL_API, + nullptr, + 0, + get_buffers().rccl_api_trace); + + if(_status != ROCPROFILER_STATUS_ERROR_NOT_IMPLEMENTED) + ROCPROFILER_CALL(_status, "buffer tracing service for rccl api configure"); + } + if(tool::get_config().counter_collection) { ROCPROFILER_CALL( @@ -1557,6 +1589,7 @@ tool_fini(void* /*tool_data*/) counter_collection_buffered_output_t{tool::get_config().counter_collection}; auto scratch_memory_output = scratch_memory_buffered_output_t{tool::get_config().scratch_memory_trace}; + auto rccl_output = rccl_buffered_output_t{tool::get_config().rccl_api_trace}; auto node_id_sort = [](const auto& lhs, const auto& rhs) { return lhs.node_id < rhs.node_id; }; @@ -1579,6 +1612,7 @@ tool_fini(void* /*tool_data*/) generate_output(hip_output, contributions); generate_output(memory_copy_output, contributions); generate_output(marker_output, contributions); + generate_output(rccl_output, contributions); generate_output(counters_output, contributions); generate_output(scratch_memory_output, contributions); @@ -1601,7 +1635,8 @@ tool_fini(void* /*tool_data*/) &memory_copy_output.element_data, &counters_output.element_data, &marker_output.element_data, - &scratch_memory_output.element_data); + &scratch_memory_output.element_data, + &rccl_output.element_data); } if(tool::get_config().pftrace_output) @@ -1614,7 +1649,8 @@ tool_fini(void* /*tool_data*/) &kernel_dispatch_output.element_data, &memory_copy_output.element_data, &marker_output.element_data, - &scratch_memory_output.element_data); + &scratch_memory_output.element_data, + &rccl_output.element_data); } if(tool::get_config().otf2_output) @@ -1627,7 +1663,8 @@ tool_fini(void* /*tool_data*/) &kernel_dispatch_output.element_data, &memory_copy_output.element_data, &marker_output.element_data, - &scratch_memory_output.element_data); + &scratch_memory_output.element_data, + &rccl_output.element_data); } if(tool::get_config().summary_output) @@ -1644,6 +1681,7 @@ tool_fini(void* /*tool_data*/) destroy_output(marker_output); destroy_output(counters_output); destroy_output(scratch_memory_output); + destroy_output(rccl_output); fini_tool_table(); if(destructors) diff --git a/source/lib/rocprofiler-sdk/CMakeLists.txt b/source/lib/rocprofiler-sdk/CMakeLists.txt index d43e9f988c..aacd9281e8 100644 --- a/source/lib/rocprofiler-sdk/CMakeLists.txt +++ b/source/lib/rocprofiler-sdk/CMakeLists.txt @@ -47,12 +47,14 @@ add_subdirectory(thread_trace) add_subdirectory(tracing) add_subdirectory(kernel_dispatch) add_subdirectory(page_migration) +add_subdirectory(rccl) add_subdirectory(details) target_link_libraries( rocprofiler-object-library PUBLIC rocprofiler-sdk::rocprofiler-headers rocprofiler-sdk::rocprofiler-hip-nolink rocprofiler-sdk::rocprofiler-hsa-runtime-nolink + rocprofiler-sdk::rocprofiler-rccl-nolink PRIVATE rocprofiler-sdk::rocprofiler-build-flags rocprofiler-sdk::rocprofiler-memcheck rocprofiler-sdk::rocprofiler-common-library diff --git a/source/lib/rocprofiler-sdk/buffer_tracing.cpp b/source/lib/rocprofiler-sdk/buffer_tracing.cpp index 1ed0f16129..de7d17534b 100644 --- a/source/lib/rocprofiler-sdk/buffer_tracing.cpp +++ b/source/lib/rocprofiler-sdk/buffer_tracing.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include "lib/common/logging.hpp" @@ -36,6 +37,7 @@ #include "lib/rocprofiler-sdk/kernel_dispatch/kernel_dispatch.hpp" #include "lib/rocprofiler-sdk/marker/marker.hpp" #include "lib/rocprofiler-sdk/page_migration/page_migration.hpp" +#include "lib/rocprofiler-sdk/rccl/rccl.hpp" #include "lib/rocprofiler-sdk/registration.hpp" #include @@ -82,6 +84,7 @@ ROCPROFILER_BUFFER_TRACING_KIND_STRING(KERNEL_DISPATCH) ROCPROFILER_BUFFER_TRACING_KIND_STRING(PAGE_MIGRATION) ROCPROFILER_BUFFER_TRACING_KIND_STRING(SCRATCH_MEMORY) ROCPROFILER_BUFFER_TRACING_KIND_STRING(CORRELATION_ID_RETIREMENT) +ROCPROFILER_BUFFER_TRACING_KIND_STRING(RCCL_API) template std::pair @@ -92,6 +95,19 @@ get_kind_name(rocprofiler_buffer_tracing_kind_t kind, std::index_sequence 0) return get_kind_name(kind, std::index_sequence{}); return {nullptr, 0}; } + +auto +get_unsupported() +{ + auto unsupported = std::unordered_set{}; + +#if ROCPROFILER_SDK_RCCL_HAS_API_TRACE == 0 + // Built against RCCL which does not support API tracing + unsupported.emplace(ROCPROFILER_BUFFER_TRACING_RCCL_API); +#endif + + return unsupported; +} } // namespace } // namespace buffer_tracing } // namespace rocprofiler @@ -104,10 +120,11 @@ rocprofiler_configure_buffer_tracing_service(rocprofiler_context_id_t size_t operations_count, rocprofiler_buffer_id_t buffer_id) { + static auto unsupported = ::rocprofiler::buffer_tracing::get_unsupported(); + if(rocprofiler::registration::get_init_status() > -1) return ROCPROFILER_STATUS_ERROR_CONFIGURATION_LOCKED; - static auto unsupported = std::unordered_set{}; if(unsupported.count(kind) > 0) return ROCPROFILER_STATUS_ERROR_NOT_IMPLEMENTED; auto* ctx = rocprofiler::context::get_mutable_registered_context(context_id); @@ -221,6 +238,11 @@ rocprofiler_query_buffer_tracing_kind_operation_name(rocprofiler_buffer_tracing_ val = rocprofiler::marker::name_by_id(operation); break; } + case ROCPROFILER_BUFFER_TRACING_RCCL_API: + { + val = rocprofiler::rccl::name_by_id(operation); + break; + } case ROCPROFILER_BUFFER_TRACING_HIP_RUNTIME_API: { val = rocprofiler::hip::name_by_id(operation); @@ -332,6 +354,11 @@ rocprofiler_iterate_buffer_tracing_kind_operations( ops = rocprofiler::marker::get_ids(); break; } + case ROCPROFILER_BUFFER_TRACING_RCCL_API: + { + ops = rocprofiler::rccl::get_ids(); + break; + } case ROCPROFILER_BUFFER_TRACING_HIP_RUNTIME_API: { ops = rocprofiler::hip::get_ids(); diff --git a/source/lib/rocprofiler-sdk/callback_tracing.cpp b/source/lib/rocprofiler-sdk/callback_tracing.cpp index 8bb966a82d..02246191d1 100644 --- a/source/lib/rocprofiler-sdk/callback_tracing.cpp +++ b/source/lib/rocprofiler-sdk/callback_tracing.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include "lib/rocprofiler-sdk/code_object/code_object.hpp" @@ -36,6 +37,7 @@ #include "lib/rocprofiler-sdk/hsa/scratch_memory.hpp" #include "lib/rocprofiler-sdk/kernel_dispatch/kernel_dispatch.hpp" #include "lib/rocprofiler-sdk/marker/marker.hpp" +#include "lib/rocprofiler-sdk/rccl/rccl.hpp" #include "lib/rocprofiler-sdk/registration.hpp" #include @@ -79,6 +81,7 @@ ROCPROFILER_CALLBACK_TRACING_KIND_STRING(CODE_OBJECT) ROCPROFILER_CALLBACK_TRACING_KIND_STRING(SCRATCH_MEMORY) ROCPROFILER_CALLBACK_TRACING_KIND_STRING(KERNEL_DISPATCH) ROCPROFILER_CALLBACK_TRACING_KIND_STRING(MEMORY_COPY) +ROCPROFILER_CALLBACK_TRACING_KIND_STRING(RCCL_API) template std::pair @@ -89,6 +92,19 @@ get_kind_name(rocprofiler_callback_tracing_kind_t kind, std::index_sequence 0) return get_kind_name(kind, std::index_sequence{}); return {nullptr, 0}; } + +auto +get_unsupported() +{ + auto unsupported = std::unordered_set{}; + +#if ROCPROFILER_SDK_RCCL_HAS_API_TRACE == 0 + // Built against RCCL which does not support API tracing + unsupported.emplace(ROCPROFILER_CALLBACK_TRACING_RCCL_API); +#endif + + return unsupported; +} } // namespace } // namespace callback_tracing } // namespace rocprofiler @@ -102,9 +118,13 @@ rocprofiler_configure_callback_tracing_service(rocprofiler_context_id_t rocprofiler_callback_tracing_cb_t callback, void* callback_args) { + static auto unsupported = ::rocprofiler::callback_tracing::get_unsupported(); + if(rocprofiler::registration::get_init_status() > -1) return ROCPROFILER_STATUS_ERROR_CONFIGURATION_LOCKED; + if(unsupported.count(kind) > 0) return ROCPROFILER_STATUS_ERROR_NOT_IMPLEMENTED; + auto* ctx = rocprofiler::context::get_mutable_registered_context(context_id); if(!ctx) return ROCPROFILER_STATUS_ERROR_CONTEXT_NOT_FOUND; @@ -200,6 +220,11 @@ rocprofiler_query_callback_tracing_kind_operation_name(rocprofiler_callback_trac val = rocprofiler::marker::name_by_id(operation); break; } + case ROCPROFILER_CALLBACK_TRACING_RCCL_API: + { + val = rocprofiler::rccl::name_by_id(operation); + break; + } case ROCPROFILER_CALLBACK_TRACING_HIP_RUNTIME_API: { val = rocprofiler::hip::name_by_id(operation); @@ -309,6 +334,11 @@ rocprofiler_iterate_callback_tracing_kind_operations( ops = rocprofiler::marker::get_ids(); break; } + case ROCPROFILER_CALLBACK_TRACING_RCCL_API: + { + ops = rocprofiler::rccl::get_ids(); + break; + } case ROCPROFILER_CALLBACK_TRACING_HIP_RUNTIME_API: { ops = rocprofiler::hip::get_ids(); @@ -463,6 +493,7 @@ rocprofiler_iterate_callback_tracing_kind_operation_args( case ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT: case ROCPROFILER_CALLBACK_TRACING_KERNEL_DISPATCH: case ROCPROFILER_CALLBACK_TRACING_MEMORY_COPY: + case ROCPROFILER_CALLBACK_TRACING_RCCL_API: { return ROCPROFILER_STATUS_ERROR_NOT_IMPLEMENTED; } diff --git a/source/lib/rocprofiler-sdk/intercept_table.cpp b/source/lib/rocprofiler-sdk/intercept_table.cpp index 12da7daed2..b3507e1519 100644 --- a/source/lib/rocprofiler-sdk/intercept_table.cpp +++ b/source/lib/rocprofiler-sdk/intercept_table.cpp @@ -33,6 +33,7 @@ #include #include +#include "lib/rocprofiler-sdk/rccl/rccl.hpp" #include #include @@ -55,7 +56,8 @@ constexpr auto intercept_library_seq = library_sequence_t{}; + ROCPROFILER_MARKER_NAME_TABLE, + ROCPROFILER_RCCL_TABLE>{}; // check that intercept_library_seq is up to date static_assert((1 << (intercept_library_seq.size() - 1)) == ROCPROFILER_TABLE_LAST, @@ -185,6 +187,11 @@ template void notify_intercept_table_registration(rocprofiler_intercept_table_t, uint64_t, uint64_t, std::tuple); + +template void notify_intercept_table_registration(rocprofiler_intercept_table_t, + uint64_t, + uint64_t, + std::tuple); } // namespace intercept_table } // namespace rocprofiler diff --git a/source/lib/rocprofiler-sdk/internal_threading.cpp b/source/lib/rocprofiler-sdk/internal_threading.cpp index 470a1985df..af9c389c38 100644 --- a/source/lib/rocprofiler-sdk/internal_threading.cpp +++ b/source/lib/rocprofiler-sdk/internal_threading.cpp @@ -121,7 +121,8 @@ using creation_notifier_cb_t = void (*)(rocprofiler_runtime_library_t, void*); constexpr auto creation_notifier_library_seq = library_sequence_t{}; + ROCPROFILER_MARKER_LIBRARY, + ROCPROFILER_RCCL_LIBRARY>{}; // check that creation_notifier_library_seq is up to date static_assert((1 << (creation_notifier_library_seq.size() - 1)) == ROCPROFILER_LIBRARY_LAST, diff --git a/source/lib/rocprofiler-sdk/rccl/CMakeLists.txt b/source/lib/rocprofiler-sdk/rccl/CMakeLists.txt new file mode 100644 index 0000000000..89b8aedf31 --- /dev/null +++ b/source/lib/rocprofiler-sdk/rccl/CMakeLists.txt @@ -0,0 +1,7 @@ +set(ROCPROFILER_LIB_RCCL_SOURCES abi.cpp rccl.cpp) +set(ROCPROFILER_LIB_RCCL_HEADERS defines.hpp rccl.hpp) + +target_sources(rocprofiler-object-library PRIVATE ${ROCPROFILER_LIB_RCCL_SOURCES} + ${ROCPROFILER_LIB_RCCL_HEADERS}) + +add_subdirectory(details) diff --git a/source/lib/rocprofiler-sdk/rccl/abi.cpp b/source/lib/rocprofiler-sdk/rccl/abi.cpp new file mode 100644 index 0000000000..a834cd9259 --- /dev/null +++ b/source/lib/rocprofiler-sdk/rccl/abi.cpp @@ -0,0 +1,82 @@ +// MIT License +// +// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "lib/rocprofiler-sdk/rccl/rccl.hpp" + +#if ROCPROFILER_SDK_RCCL_HAS_API_TRACE > 0 + +# include "lib/common/abi.hpp" +# include "lib/common/defines.hpp" + +# include +# include + +namespace rocprofiler +{ +namespace rccl +{ +static_assert(RCCL_API_TRACE_VERSION_MAJOR == 0, "Major version updated for RCCL dispatch table"); +static_assert(RCCL_API_TRACE_VERSION_PATCH == 0, "Patch version updated for RCCL dispatch table"); + +ROCP_SDK_ENFORCE_ABI_VERSIONING(rcclApiFuncTable, 37) + +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclAllGather_fn, 0) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclAllReduce_fn, 1) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclAllToAll_fn, 2) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclAllToAllv_fn, 3) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclBroadcast_fn, 4) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclGather_fn, 5) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclReduce_fn, 6) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclReduceScatter_fn, 7) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclScatter_fn, 8) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclSend_fn, 9) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclRecv_fn, 10) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclRedOpCreatePreMulSum_fn, 11) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclRedOpDestroy_fn, 12) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclGroupStart_fn, 13) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclGroupEnd_fn, 14) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclGetVersion_fn, 15) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclGetUniqueId_fn, 16) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclCommInitRank_fn, 17) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclCommInitAll_fn, 18) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclCommInitRankConfig_fn, 19) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclCommFinalize_fn, 20) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclCommDestroy_fn, 21) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclCommAbort_fn, 22) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclCommSplit_fn, 23) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclGetErrorString_fn, 24) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclGetLastError_fn, 25) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclCommGetAsyncError_fn, 26) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclCommCount_fn, 27) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclCommCuDevice_fn, 28) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclCommUserRank_fn, 29) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclMemAlloc_fn, 30) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclMemFree_fn, 31) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, mscclLoadAlgo_fn, 32) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, mscclRunAlgo_fn, 33) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, mscclUnloadAlgo_fn, 34) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclCommRegister_fn, 35) +ROCP_SDK_ENFORCE_ABI(rcclApiFuncTable, ncclCommDeregister_fn, 36) +} // namespace rccl +} // namespace rocprofiler + +#endif diff --git a/source/lib/rocprofiler-sdk/rccl/defines.hpp b/source/lib/rocprofiler-sdk/rccl/defines.hpp new file mode 100644 index 0000000000..24fac2dce4 --- /dev/null +++ b/source/lib/rocprofiler-sdk/rccl/defines.hpp @@ -0,0 +1,212 @@ +// MIT License +// +// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#pragma once + +#include "lib/common/defines.hpp" + +#define RCCL_API_INFO_DEFINITION_0(RCCL_TABLE, RCCL_API_ID, RCCL_FUNC, RCCL_FUNC_PTR) \ + namespace rocprofiler \ + { \ + namespace rccl \ + { \ + template <> \ + struct rccl_api_info : rccl_domain_info \ + { \ + static constexpr auto table_idx = RCCL_TABLE; \ + static constexpr auto operation_idx = RCCL_API_ID; \ + static constexpr auto name = #RCCL_FUNC; \ + \ + using domain_type = rccl_domain_info; \ + using this_type = rccl_api_info; \ + using base_type = rccl_api_impl; \ + \ + using domain_type::callback_domain_idx; \ + using domain_type::buffered_domain_idx; \ + using domain_type::args_type; \ + using domain_type::retval_type; \ + using domain_type::callback_data_type; \ + \ + static constexpr auto offset() \ + { \ + return offsetof(rccl_table_lookup::type, RCCL_FUNC_PTR); \ + } \ + \ + static_assert(offsetof(rccl_table_lookup::type, RCCL_FUNC_PTR) == \ + (sizeof(size_t) + (operation_idx * sizeof(void*))), \ + "ABI error for " #RCCL_FUNC); \ + \ + static auto& get_table() { return rccl_table_lookup{}(); } \ + \ + template \ + static auto& get_table(TableT& _v) \ + { \ + return rccl_table_lookup{}(_v); \ + } \ + \ + template \ + static auto& get_table_func(TableT& _table) \ + { \ + if constexpr(std::is_pointer::value) \ + { \ + assert(_table != nullptr && "nullptr to MARKER table for " #RCCL_FUNC \ + " function"); \ + return _table->RCCL_FUNC_PTR; \ + } \ + else \ + { \ + return _table.RCCL_FUNC_PTR; \ + } \ + } \ + \ + static auto& get_table_func() { return get_table_func(get_table()); } \ + \ + template \ + static auto& get_api_data_args(DataT& _data) \ + { \ + return _data.RCCL_FUNC; \ + } \ + \ + template \ + static auto get_functor(RetT (*)(Args...)) \ + { \ + return &base_type::functor; \ + } \ + \ + static std::vector as_arg_addr(callback_data_type) { return std::vector{}; } \ + \ + static std::vector as_arg_list(callback_data_type, int32_t) \ + { \ + return {}; \ + } \ + }; \ + } \ + } + +#define RCCL_API_INFO_DEFINITION_V(RCCL_TABLE, RCCL_API_ID, RCCL_FUNC, RCCL_FUNC_PTR, ...) \ + namespace rocprofiler \ + { \ + namespace rccl \ + { \ + template <> \ + struct rccl_api_info : rccl_domain_info \ + { \ + static constexpr auto table_idx = RCCL_TABLE; \ + static constexpr auto operation_idx = RCCL_API_ID; \ + static constexpr auto name = #RCCL_FUNC; \ + \ + using domain_type = rccl_domain_info; \ + using this_type = rccl_api_info; \ + using base_type = rccl_api_impl; \ + \ + static constexpr auto callback_domain_idx = domain_type::callback_domain_idx; \ + static constexpr auto buffered_domain_idx = domain_type::buffered_domain_idx; \ + \ + using domain_type::args_type; \ + using domain_type::retval_type; \ + using domain_type::callback_data_type; \ + \ + static constexpr auto offset() \ + { \ + return offsetof(rccl_table_lookup::type, RCCL_FUNC_PTR); \ + } \ + \ + static_assert(offsetof(rccl_table_lookup::type, RCCL_FUNC_PTR) == \ + (sizeof(size_t) + (operation_idx * sizeof(void*))), \ + "ABI error for " #RCCL_FUNC); \ + \ + static auto& get_table() { return rccl_table_lookup{}(); } \ + \ + template \ + static auto& get_table(TableT& _v) \ + { \ + return rccl_table_lookup{}(_v); \ + } \ + \ + template \ + static auto& get_table_func(TableT& _table) \ + { \ + if constexpr(std::is_pointer::value) \ + { \ + assert(_table != nullptr && "nullptr to MARKER table for " #RCCL_FUNC \ + " function"); \ + return _table->RCCL_FUNC_PTR; \ + } \ + else \ + { \ + return _table.RCCL_FUNC_PTR; \ + } \ + } \ + \ + static auto& get_table_func() { return get_table_func(get_table()); } \ + \ + template \ + static auto& get_api_data_args(DataT& _data) \ + { \ + return _data.RCCL_FUNC; \ + } \ + \ + template \ + static auto get_functor(RetT (*)(Args...)) \ + { \ + return &base_type::functor; \ + } \ + \ + static std::vector as_arg_addr(callback_data_type trace_data) \ + { \ + return std::vector{ \ + GET_ADDR_MEMBER_FIELDS(get_api_data_args(trace_data.args), __VA_ARGS__)}; \ + } \ + }; \ + } \ + } + +#define RCCL_API_TABLE_LOOKUP_DEFINITION(TABLE_ID, TYPE) \ + namespace rocprofiler \ + { \ + namespace rccl \ + { \ + namespace \ + { \ + template <> \ + auto* get_table() \ + { \ + return get_table_impl(); \ + } \ + } \ + \ + template <> \ + struct rccl_table_lookup \ + { \ + using type = TYPE; \ + auto& operator()(type& _v) const { return _v; } \ + auto& operator()(type* _v) const { return *_v; } \ + auto& operator()() const { return (*this)(get_table()); } \ + }; \ + \ + template <> \ + struct rccl_table_id_lookup \ + { \ + static constexpr auto value = TABLE_ID; \ + }; \ + } \ + } diff --git a/source/lib/rocprofiler-sdk/rccl/details/CMakeLists.txt b/source/lib/rocprofiler-sdk/rccl/details/CMakeLists.txt new file mode 100644 index 0000000000..d2f12fefc4 --- /dev/null +++ b/source/lib/rocprofiler-sdk/rccl/details/CMakeLists.txt @@ -0,0 +1,8 @@ +# +# +# +set(ROCPROFILER_LIB_RCCL_DETAILS_SOURCES) +set(ROCPROFILER_LIB_RCCL_DETAILS_HEADERS api_trace.h) + +target_sources(rocprofiler-object-library PRIVATE ${ROCPROFILER_LIB_RCCL_DETAILS_SOURCES} + ${ROCPROFILER_LIB_RCCL_DETAILS_HEADERS}) diff --git a/source/lib/rocprofiler-sdk/rccl/details/api_trace.h b/source/lib/rocprofiler-sdk/rccl/details/api_trace.h new file mode 100644 index 0000000000..a594a29227 --- /dev/null +++ b/source/lib/rocprofiler-sdk/rccl/details/api_trace.h @@ -0,0 +1,254 @@ +// MIT License +// +// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#pragma once + +#include + +#include +#include + +// should only be increased if fundamental changes to dispatch table(s) +#define RCCL_API_TRACE_VERSION_MAJOR 0 + +// should be increased every time new members are added to existing dispatch tables +#define RCCL_API_TRACE_VERSION_PATCH 0 + +#if !defined(RCCL_EXTERN_C_INIT) +# ifdef __cplusplus +# define RCCL_EXTERN_C_INIT extern "C" { +# else +# define RCCL_EXTERN_C_INIT +# endif +#endif + +#if !defined(RCCL_EXTERN_C_FINI) +# ifdef __cplusplus +# define RCCL_EXTERN_C_FINI } +# else +# define RCCL_EXTERN_C_FINI +# endif +#endif + +RCCL_EXTERN_C_INIT + +typedef uint64_t rccl_range_id_t; +typedef ncclResult_t (*ncclAllGather_fn_t)(const void* sendbuff, + void* recvbuff, + size_t sendcount, + ncclDataType_t datatype, + ncclComm_t comm, + hipStream_t stream); +typedef ncclResult_t (*ncclAllReduce_fn_t)(const void* sendbuff, + void* recvbuff, + size_t count, + ncclDataType_t datatype, + ncclRedOp_t op, + struct ncclComm* comm, + hipStream_t stream); +typedef ncclResult_t (*ncclAllToAll_fn_t)(const void* sendbuff, + void* recvbuff, + size_t count, + ncclDataType_t datatype, + ncclComm_t comm, + hipStream_t stream); +typedef ncclResult_t (*ncclAllToAllv_fn_t)(const void* sendbuff, + const size_t sendcounts[], + const size_t sdispls[], + void* recvbuff, + const size_t recvcounts[], + const size_t rdispls[], + ncclDataType_t datatype, + ncclComm_t comm, + hipStream_t stream); +typedef ncclResult_t (*ncclBroadcast_fn_t)(const void* sendbuff, + void* recvbuff, + size_t count, + ncclDataType_t datatype, + int root, + ncclComm_t comm, + hipStream_t stream); +typedef ncclResult_t (*ncclGather_fn_t)(const void* sendbuff, + void* recvbuff, + size_t sendcount, + ncclDataType_t datatype, + int root, + ncclComm_t comm, + hipStream_t stream); +typedef ncclResult_t (*ncclReduce_fn_t)(const void* sendbuff, + void* recvbuff, + size_t count, + ncclDataType_t datatype, + ncclRedOp_t op, + int root, + ncclComm_t comm, + hipStream_t stream); +typedef ncclResult_t (*ncclReduceScatter_fn_t)(const void* sendbuff, + void* recvbuff, + size_t recvcount, + ncclDataType_t datatype, + ncclRedOp_t op, + struct ncclComm* comm, + hipStream_t stream); +typedef ncclResult_t (*ncclScatter_fn_t)(const void* sendbuff, + void* recvbuff, + size_t recvcount, + ncclDataType_t datatype, + int root, + ncclComm_t comm, + hipStream_t stream); +typedef ncclResult_t (*ncclSend_fn_t)(const void* sendbuff, + size_t count, + ncclDataType_t datatype, + int peer, + ncclComm_t comm, + hipStream_t stream); +typedef ncclResult_t (*ncclRecv_fn_t)(void* recvbuff, + size_t count, + ncclDataType_t datatype, + int peer, + ncclComm_t comm, + hipStream_t stream); +typedef ncclResult_t (*ncclRedOpCreatePreMulSum_fn_t)(ncclRedOp_t* op, + void* scalar, + ncclDataType_t datatype, + ncclScalarResidence_t residence, + ncclComm_t comm); +typedef ncclResult_t (*ncclRedOpDestroy_fn_t)(ncclRedOp_t op, ncclComm_t comm); +typedef ncclResult_t (*ncclGroupStart_fn_t)(); +typedef ncclResult_t (*ncclGroupEnd_fn_t)(); +typedef ncclResult_t (*ncclGetVersion_fn_t)(int* version); +typedef ncclResult_t (*ncclGetUniqueId_fn_t)(ncclUniqueId* out); + +typedef ncclResult_t (*ncclCommInitRank_fn_t)(ncclComm_t* newcomm, + int nranks, + ncclUniqueId commId, + int myrank); + +typedef ncclResult_t (*ncclCommInitAll_fn_t)(ncclComm_t* comms, int ndev, const int* devlist); + +typedef ncclResult_t (*ncclCommInitRankConfig_fn_t)(ncclComm_t* comm, + int nranks, + ncclUniqueId commId, + int myrank, + ncclConfig_t* config); + +typedef ncclResult_t (*ncclCommFinalize_fn_t)(ncclComm_t comm); + +typedef ncclResult_t (*ncclCommDestroy_fn_t)(ncclComm_t comm); + +typedef ncclResult_t (*ncclCommAbort_fn_t)(ncclComm_t comm); + +typedef ncclResult_t (*ncclCommSplit_fn_t)(ncclComm_t comm, + int color, + int key, + ncclComm_t* newcomm, + ncclConfig_t* config); + +typedef const char* (*ncclGetErrorString_fn_t)(ncclResult_t code); + +typedef const char* (*ncclGetLastError_fn_t)(const ncclComm_t comm); + +typedef ncclResult_t (*ncclCommGetAsyncError_fn_t)(ncclComm_t comm, ncclResult_t* asyncError); + +typedef ncclResult_t (*ncclCommCount_fn_t)(const ncclComm_t comm, int* count); + +typedef ncclResult_t (*ncclCommCuDevice_fn_t)(const ncclComm_t comm, int* devid); + +typedef ncclResult_t (*ncclCommUserRank_fn_t)(const ncclComm_t comm, int* rank); + +typedef ncclResult_t (*ncclMemAlloc_fn_t)(void** ptr, size_t size); + +typedef ncclResult_t (*ncclMemFree_fn_t)(void* ptr); + +typedef ncclResult_t (*mscclLoadAlgo_fn_t)(const char* mscclAlgoFilePath, + mscclAlgoHandle_t* mscclAlgoHandle, + int rank); + +typedef ncclResult_t (*mscclRunAlgo_fn_t)(const void* sendBuff, + const size_t sendCounts[], + const size_t sDisPls[], + void* recvBuff, + const size_t recvCounts[], + const size_t rDisPls[], + size_t count, + ncclDataType_t dataType, + int root, + int peer, + ncclRedOp_t op, + mscclAlgoHandle_t mscclAlgoHandle, + ncclComm_t comm, + hipStream_t stream); + +typedef ncclResult_t (*mscclUnloadAlgo_fn_t)(mscclAlgoHandle_t mscclAlgoHandle); + +typedef ncclResult_t (*ncclCommRegister_fn_t)(const ncclComm_t comm, + void* buff, + size_t size, + void** handle); + +typedef ncclResult_t (*ncclCommDeregister_fn_t)(const ncclComm_t comm, void* handle); + +typedef struct rcclApiFuncTable +{ + uint64_t size; + ncclAllGather_fn_t ncclAllGather_fn; + ncclAllReduce_fn_t ncclAllReduce_fn; + ncclAllToAll_fn_t ncclAllToAll_fn; + ncclAllToAllv_fn_t ncclAllToAllv_fn; + ncclBroadcast_fn_t ncclBroadcast_fn; + ncclGather_fn_t ncclGather_fn; + ncclReduce_fn_t ncclReduce_fn; + ncclReduceScatter_fn_t ncclReduceScatter_fn; + ncclScatter_fn_t ncclScatter_fn; + ncclSend_fn_t ncclSend_fn; + ncclRecv_fn_t ncclRecv_fn; + ncclRedOpCreatePreMulSum_fn_t ncclRedOpCreatePreMulSum_fn; + ncclRedOpDestroy_fn_t ncclRedOpDestroy_fn; + ncclGroupStart_fn_t ncclGroupStart_fn; + ncclGroupEnd_fn_t ncclGroupEnd_fn; + ncclGetVersion_fn_t ncclGetVersion_fn; + ncclGetUniqueId_fn_t ncclGetUniqueId_fn; + ncclCommInitRank_fn_t ncclCommInitRank_fn; + ncclCommInitAll_fn_t ncclCommInitAll_fn; + ncclCommInitRankConfig_fn_t ncclCommInitRankConfig_fn; + ncclCommFinalize_fn_t ncclCommFinalize_fn; + ncclCommDestroy_fn_t ncclCommDestroy_fn; + ncclCommAbort_fn_t ncclCommAbort_fn; + ncclCommSplit_fn_t ncclCommSplit_fn; + ncclGetErrorString_fn_t ncclGetErrorString_fn; + ncclGetLastError_fn_t ncclGetLastError_fn; + ncclCommGetAsyncError_fn_t ncclCommGetAsyncError_fn; + ncclCommCount_fn_t ncclCommCount_fn; + ncclCommCuDevice_fn_t ncclCommCuDevice_fn; + ncclCommUserRank_fn_t ncclCommUserRank_fn; + ncclMemAlloc_fn_t ncclMemAlloc_fn; + ncclMemFree_fn_t ncclMemFree_fn; + mscclLoadAlgo_fn_t mscclLoadAlgo_fn; + mscclRunAlgo_fn_t mscclRunAlgo_fn; + mscclUnloadAlgo_fn_t mscclUnloadAlgo_fn; + ncclCommRegister_fn_t ncclCommRegister_fn; + ncclCommDeregister_fn_t ncclCommDeregister_fn; + +} rcclApiFuncTable; + +RCCL_EXTERN_C_FINI diff --git a/source/lib/rocprofiler-sdk/rccl/rccl.cpp b/source/lib/rocprofiler-sdk/rccl/rccl.cpp new file mode 100644 index 0000000000..7b1df51458 --- /dev/null +++ b/source/lib/rocprofiler-sdk/rccl/rccl.cpp @@ -0,0 +1,557 @@ +// MIT License +// +// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "lib/rocprofiler-sdk/rccl/rccl.hpp" +#include "lib/common/defines.hpp" +#include "lib/common/static_object.hpp" +#include "lib/common/utility.hpp" +#include "lib/rocprofiler-sdk/buffer.hpp" +#include "lib/rocprofiler-sdk/context/context.hpp" +#include "lib/rocprofiler-sdk/hip/hip.hpp" +#include "lib/rocprofiler-sdk/hip/utils.hpp" +#include "lib/rocprofiler-sdk/registration.hpp" +#include "lib/rocprofiler-sdk/tracing/tracing.hpp" + +#include +#include +#include +#include + +#include +#include +// must be included after runtime api +#include + +#include +#include +#include +#include + +namespace rocprofiler +{ +namespace rccl +{ +namespace +{ +struct null_type +{}; + +template +auto +get_default_retval() +{ + if constexpr(std::is_pointer::value) + { + Tp v = nullptr; + return v; + } + else if constexpr(std::is_same::value) + return ncclInternalError; // No unknown error, ok? + else if constexpr(std::is_same::value) + return "UnknownString"; // ok? + else + static_assert(std::is_empty::value, "Error! unsupported return type"); +} + +template +void +set_data_retval(DataT& _data, Tp _val) +{ + if constexpr(std::is_same::value) + { + _data.ncclResult_t_retval = _val; + } + else if constexpr(std::is_same::value) + { + _data.const_charp_retval = _val; + } + else + { + static_assert(std::is_empty::value, "Error! unsupported return type"); + } +} + +template +Tp* +get_table_impl() +{ + static auto*& _v = common::static_object::construct(common::init_public_api_struct(Tp{})); + return _v; +} + +template +auto* +get_table(); + +} // namespace + +template +template +auto +rccl_api_impl::set_data_args(DataArgsT& _data_args, Args... args) +{ + if constexpr(sizeof...(Args) == 0) + _data_args.no_args.empty = '\0'; + else + _data_args = DataArgsT{args...}; +} + +template +template +auto +rccl_api_impl::exec(FuncT&& _func, Args&&... args) +{ + using return_type = std::decay_t>; + + if(_func) + { + if constexpr(std::is_void::value) + { + _func(std::forward(args)...); + return null_type{}; + } + else + { + return _func(std::forward(args)...); + } + } + + using info_type = rccl_api_info; + ROCP_ERROR << "nullptr to next hip function for " << info_type::name << " (" + << info_type::operation_idx << ")"; + + return get_default_retval(); +} + +template +template +RetT +rccl_api_impl::functor(Args... args) +{ + using info_type = rccl_api_info; + using callback_api_data_t = typename rccl_domain_info::callback_data_type; + using buffered_api_data_t = typename rccl_domain_info::buffer_data_type; + + constexpr auto external_corr_id_domain_idx = + rccl_domain_info::external_correlation_id_domain_idx; + + if(registration::get_fini_status() != 0) + { + [[maybe_unused]] auto _ret = exec(info_type::get_table_func(), std::forward(args)...); + if constexpr(!std::is_void::value) + return _ret; + else + return; + } + + constexpr auto ref_count = 2; + auto thr_id = common::get_tid(); + auto callback_contexts = tracing::callback_context_data_vec_t{}; + auto buffered_contexts = tracing::buffered_context_data_vec_t{}; + auto external_corr_ids = tracing::external_correlation_id_map_t{}; + + tracing::populate_contexts(info_type::callback_domain_idx, + info_type::buffered_domain_idx, + info_type::operation_idx, + callback_contexts, + buffered_contexts, + external_corr_ids); + + if(callback_contexts.empty() && buffered_contexts.empty()) + { + [[maybe_unused]] auto _ret = exec(info_type::get_table_func(), std::forward(args)...); + if constexpr(!std::is_void::value) + return _ret; + else + return; + } + + auto buffer_record = common::init_public_api_struct(buffered_api_data_t{}); + auto tracer_data = common::init_public_api_struct(callback_api_data_t{}); + auto* corr_id = tracing::correlation_service::construct(ref_count); + auto internal_corr_id = corr_id->internal; + + tracing::populate_external_correlation_ids(external_corr_ids, + thr_id, + external_corr_id_domain_idx, + info_type::operation_idx, + internal_corr_id); + + // invoke the callbacks + if(!callback_contexts.empty()) + { + set_data_args(info_type::get_api_data_args(tracer_data.args), std::forward(args)...); + + tracing::execute_phase_enter_callbacks(callback_contexts, + thr_id, + internal_corr_id, + external_corr_ids, + info_type::callback_domain_idx, + info_type::operation_idx, + tracer_data); + } + + // enter callback may update the external correlation id field + tracing::update_external_correlation_ids( + external_corr_ids, thr_id, external_corr_id_domain_idx); + + // record the start timestamp as close to the function call as possible + if(!buffered_contexts.empty()) + { + buffer_record.start_timestamp = common::timestamp_ns(); + } + + // decrement the reference count before invoking + corr_id->sub_ref_count(); + + auto _ret = exec(info_type::get_table_func(), std::forward(args)...); + + // record the end timestamp as close to the function call as possible + if(!buffered_contexts.empty()) + { + buffer_record.end_timestamp = common::timestamp_ns(); + } + + if(!callback_contexts.empty()) + { + set_data_retval(tracer_data.retval, _ret); + + tracing::execute_phase_exit_callbacks(callback_contexts, + external_corr_ids, + info_type::callback_domain_idx, + info_type::operation_idx, + tracer_data); + } + + if(!buffered_contexts.empty()) + { + tracing::execute_buffer_record_emplace(buffered_contexts, + thr_id, + internal_corr_id, + external_corr_ids, + info_type::buffered_domain_idx, + info_type::operation_idx, + buffer_record); + } + + // decrement the reference count after usage in the callback/buffers + corr_id->sub_ref_count(); + + context::pop_latest_correlation_id(corr_id); + + if constexpr(!std::is_void::value) return _ret; +} +} // namespace rccl +} // namespace rocprofiler + +#define ROCPROFILER_LIB_ROCPROFILER_SDK_RCCL_RCCL_CPP_IMPL 1 + +// template specializations +#include "rccl.def.cpp" + +namespace rocprofiler +{ +namespace rccl +{ +namespace +{ +template +const char* +name_by_id(const uint32_t id, std::index_sequence) +{ + if(OpIdx == id) return rccl_api_info::name; + + if constexpr(sizeof...(OpIdxTail) > 0) + return name_by_id(id, std::index_sequence{}); + else + return nullptr; +} + +template +uint32_t +id_by_name(const char* name, std::index_sequence) +{ + if(std::string_view{rccl_api_info::name} == std::string_view{name}) + return rccl_api_info::operation_idx; + + if constexpr(sizeof...(OpIdxTail) > 0) + return id_by_name(name, std::index_sequence{}); + else + return rccl_domain_info::none; +} + +template +void +get_ids(std::vector& _id_list, std::index_sequence) +{ + auto _idx = rccl_api_info::operation_idx; + if(_idx < rccl_domain_info::last) _id_list.emplace_back(_idx); + + if constexpr(sizeof...(OpIdxTail) > 0) + get_ids(_id_list, std::index_sequence{}); +} + +template +void +get_names(std::vector& _name_list, std::index_sequence) +{ + auto&& _name = rccl_api_info::name; + if(_name != nullptr && strnlen(_name, 1) > 0) _name_list.emplace_back(_name); + + if constexpr(sizeof...(OpIdxTail) > 0) + get_names(_name_list, std::index_sequence{}); +} + +template +void +iterate_args(const uint32_t id, + const DataT& data, + rocprofiler_callback_tracing_operation_args_cb_t func, + int32_t max_deref, + void* user_data, + std::index_sequence) +{ + if(OpIdx == id) + { + using info_type = rccl_api_info; + auto&& arg_list = info_type::as_arg_list(data, max_deref); + auto&& arg_addr = info_type::as_arg_addr(data); + for(size_t i = 0; i < std::min(arg_list.size(), arg_addr.size()); ++i) + { + auto ret = func(info_type::callback_domain_idx, // kind + id, // operation + i, // arg_number + arg_addr.at(i), // arg_value_addr + arg_list.at(i).indirection_level, // indirection + arg_list.at(i).type, // arg_type + arg_list.at(i).name, // arg_name + arg_list.at(i).value.c_str(), // arg_value_str + arg_list.at(i).dereference_count, // num deref in str + user_data); + if(ret != 0) break; + } + return; + } + if constexpr(sizeof...(OpIdxTail) > 0) + iterate_args( + id, data, func, max_deref, user_data, std::index_sequence{}); +} + +bool +should_wrap_functor(rocprofiler_callback_tracing_kind_t _callback_domain, + rocprofiler_buffer_tracing_kind_t _buffered_domain, + int _operation) +{ + // we loop over all the *registered* contexts and see if any of them, at any point in time, + // might require callback or buffered API tracing + for(const auto& itr : context::get_registered_contexts()) + { + if(!itr) continue; + + // if there is a callback tracer enabled for the given domain and op, we need to wrap + if(itr->callback_tracer && itr->callback_tracer->domains(_callback_domain) && + itr->callback_tracer->domains(_callback_domain, _operation)) + return true; + + // if there is a buffered tracer enabled for the given domain and op, we need to wrap + if(itr->buffered_tracer && itr->buffered_tracer->domains(_buffered_domain) && + itr->buffered_tracer->domains(_buffered_domain, _operation)) + return true; + } + return false; +} + +template +void +copy_table(Tp* _orig, uint64_t _tbl_instance, std::integral_constant) +{ + using table_type = typename rccl_table_lookup::type; + + if constexpr(std::is_same::value) + { + auto _info = rccl_api_info{}; + + // make sure we don't access a field that doesn't exist in input table + if(_info.offset() >= _orig->size) return; + + // 1. get the sub-table containing the function pointer in original table + // 2. get reference to function pointer in sub-table in original table + auto& _orig_table = _info.get_table(_orig); + auto& _orig_func = _info.get_table_func(_orig_table); + // 3. get the sub-table containing the function pointer in saved table + // 4. get reference to function pointer in sub-table in saved table + // 5. save the original function in the saved table + auto& _copy_table = _info.get_table(*get_table()); + auto& _copy_func = _info.get_table_func(_copy_table); + + ROCP_FATAL_IF(_copy_func && _tbl_instance == 0) + << _info.name << " has non-null function pointer " << _copy_func + << " despite this being the first instance of the library being copies"; + + if(!_copy_func) + { + ROCP_TRACE << "copying table entry for " << _info.name; + _copy_func = _orig_func; + } + else + { + ROCP_TRACE << "skipping copying table entry for " << _info.name + << " from table instance " << _tbl_instance; + } + } +} + +template +void +update_table(Tp* _orig, std::integral_constant) +{ + using table_type = typename rccl_table_lookup::type; + + if constexpr(std::is_same::value) + { + auto _info = rccl_api_info{}; + + // make sure we don't access a field that doesn't exist in input table + if(_info.offset() >= _orig->size) return; + + // check to see if there are any contexts which enable this operation in the HIP API domain + if(!should_wrap_functor( + _info.callback_domain_idx, _info.buffered_domain_idx, _info.operation_idx)) + return; + + ROCP_TRACE << "updating table entry for " << _info.name; + + // 1. get the sub-table containing the function pointer in original table + // 2. get reference to function pointer in sub-table in original table + // 3. update function pointer with wrapper + auto& _table = _info.get_table(_orig); + auto& _func = _info.get_table_func(_table); + _func = _info.get_functor(_func); + } +} + +template +void +copy_table(Tp* _orig, uint64_t _tbl_instance, std::index_sequence) +{ + copy_table(_orig, _tbl_instance, std::integral_constant{}); + if constexpr(sizeof...(OpIdxTail) > 0) + copy_table(_orig, _tbl_instance, std::index_sequence{}); +} + +template +void +update_table(Tp* _orig, std::index_sequence) +{ + update_table(_orig, std::integral_constant{}); + if constexpr(sizeof...(OpIdxTail) > 0) + update_table(_orig, std::index_sequence{}); +} +} // namespace + +// check out the assembly here... this compiles to a switch statement +template +const char* +name_by_id(uint32_t id) +{ + return name_by_id(id, std::make_index_sequence::last>{}); +} + +template +uint32_t +id_by_name(const char* name) +{ + return id_by_name(name, std::make_index_sequence::last>{}); +} + +template +std::vector +get_ids() +{ + constexpr auto last_api_id = rccl_domain_info::last; + auto _data = std::vector{}; + _data.reserve(last_api_id); + get_ids(_data, std::make_index_sequence{}); + return _data; +} + +template +std::vector +get_names() +{ + constexpr auto last_api_id = rccl_domain_info::last; + auto _data = std::vector{}; + _data.reserve(last_api_id); + get_names(_data, std::make_index_sequence{}); + return _data; +} + +template +void +iterate_args(uint32_t id, + const rocprofiler_callback_tracing_hip_api_data_t& data, + rocprofiler_callback_tracing_operation_args_cb_t callback, + int32_t max_deref, + void* user_data) +{ + if(callback) + iterate_args(id, + data, + callback, + max_deref, + user_data, + std::make_index_sequence::last>{}); +} + +template +void +copy_table(TableT* _orig, uint64_t _tbl_instance) +{ + constexpr auto TableIdx = rccl_table_id_lookup::value; + if(_orig) + copy_table( + _orig, _tbl_instance, std::make_index_sequence::last>{}); +} + +template +void +update_table(TableT* _orig) +{ + constexpr auto TableIdx = rccl_table_id_lookup::value; + if(_orig) + update_table(_orig, std::make_index_sequence::last>{}); +} + +using rccl_api_data_t = rocprofiler_callback_tracing_rccl_api_data_t; +using rccl_op_args_cb_t = rocprofiler_callback_tracing_operation_args_cb_t; + +#define INSTANTIATE_RCCL_TABLE_FUNC(TABLE_TYPE, TABLE_IDX) \ + template void copy_table(TABLE_TYPE * _tbl, uint64_t _instv); \ + template void update_table(TABLE_TYPE * _tbl); \ + template const char* name_by_id(uint32_t); \ + template uint32_t id_by_name(const char*); \ + template std::vector get_ids(); \ + template std::vector get_names(); + +INSTANTIATE_RCCL_TABLE_FUNC(rccl_api_func_table_t, ROCPROFILER_RCCL_TABLE_ID) +} // namespace rccl +} // namespace rocprofiler diff --git a/source/lib/rocprofiler-sdk/rccl/rccl.def.cpp b/source/lib/rocprofiler-sdk/rccl/rccl.def.cpp new file mode 100644 index 0000000000..08d74b60ce --- /dev/null +++ b/source/lib/rocprofiler-sdk/rccl/rccl.def.cpp @@ -0,0 +1,109 @@ +// MIT License +// +// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "lib/rocprofiler-sdk/rccl/defines.hpp" +#include "lib/rocprofiler-sdk/rccl/rccl.hpp" + +#include +#include +#include +#include + +namespace rocprofiler +{ +namespace rccl +{ +template <> +struct rccl_domain_info +{ + using args_type = rocprofiler_rccl_api_args_t; + using retval_type = rocprofiler_rccl_api_retval_t; + using callback_data_type = rocprofiler_callback_tracing_rccl_api_data_t; + using buffer_data_type = rocprofiler_buffer_tracing_rccl_api_record_t; +}; + +template <> +struct rccl_domain_info +: rccl_domain_info +{ + using enum_type = rocprofiler_marker_core_api_id_t; + static constexpr auto callback_domain_idx = ROCPROFILER_CALLBACK_TRACING_RCCL_API; + static constexpr auto buffered_domain_idx = ROCPROFILER_BUFFER_TRACING_RCCL_API; + static constexpr auto none = ROCPROFILER_RCCL_API_ID_NONE; + static constexpr auto last = ROCPROFILER_RCCL_API_ID_LAST; + static constexpr auto external_correlation_id_domain_idx = + ROCPROFILER_EXTERNAL_CORRELATION_REQUEST_RCCL_API; +}; + +} // namespace rccl +} // namespace rocprofiler + +#if defined(ROCPROFILER_LIB_ROCPROFILER_SDK_RCCL_RCCL_CPP_IMPL) && \ + ROCPROFILER_LIB_ROCPROFILER_SDK_RCCL_RCCL_CPP_IMPL == 1 + +// clang-format off +RCCL_API_TABLE_LOOKUP_DEFINITION(ROCPROFILER_RCCL_TABLE_ID, rccl_api_func_table_t) + +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclAllGather, ncclAllGather, ncclAllGather_fn, sendbuff, recvbuff, sendcount, datatype, comm, stream) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclAllReduce, ncclAllReduce, ncclAllReduce_fn, sendbuff, recvbuff, count, datatype, op, comm, stream) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclAllToAll, ncclAllToAll, ncclAllToAll_fn, sendbuff, recvbuff, count, datatype, comm, stream) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclAllToAllv, ncclAllToAllv, ncclAllToAllv_fn, sendbuff, sendcounts, sdispls, recvbuff, recvcounts, rdispls, datatype, comm, stream) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclBroadcast, ncclBroadcast, ncclBroadcast_fn, sendbuff, recvbuff, count, datatype, root, comm, stream) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclGather, ncclGather, ncclGather_fn, sendbuff, recvbuff, sendcount, datatype, root, comm, stream) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclReduce, ncclReduce, ncclReduce_fn, sendbuff, recvbuff, count, datatype, op, root, comm, stream) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclReduceScatter, ncclReduceScatter, ncclReduceScatter_fn, sendbuff, recvbuff, recvcount, datatype, op, comm, stream) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclScatter, ncclScatter, ncclScatter_fn, sendbuff, recvbuff, recvcount, datatype, root, comm, stream) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclSend, ncclSend, ncclSend_fn, sendbuff, count, datatype, peer, comm, stream) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclRecv, ncclRecv, ncclRecv_fn, recvbuff, count, datatype, peer, comm, stream) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclRedOpCreatePreMulSum, ncclRedOpCreatePreMulSum, ncclRedOpCreatePreMulSum_fn, op, scalar, datatype, residence, comm) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclRedOpDestroy, ncclRedOpDestroy, ncclRedOpDestroy_fn, op, comm) + +RCCL_API_INFO_DEFINITION_0(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclGroupStart, ncclGroupStart, ncclGroupStart_fn) +RCCL_API_INFO_DEFINITION_0(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclGroupEnd, ncclGroupEnd, ncclGroupEnd_fn) + +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclGetVersion, ncclGetVersion, ncclGetVersion_fn, version) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclGetUniqueId, ncclGetUniqueId, ncclGetUniqueId_fn, out) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclCommInitRank, ncclCommInitRank, ncclCommInitRank_fn, newcomm, nranks, commId, myrank) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclCommInitAll, ncclCommInitAll, ncclCommInitAll_fn, comms, ndev, devlist) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclCommInitRankConfig, ncclCommInitRankConfig, ncclCommInitRankConfig_fn, comm, nranks, commId, myrank, config) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclCommFinalize, ncclCommFinalize, ncclCommFinalize_fn, comm) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclCommDestroy, ncclCommDestroy, ncclCommDestroy_fn, comm) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclCommAbort, ncclCommAbort, ncclCommAbort_fn, comm) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclCommSplit, ncclCommSplit, ncclCommSplit_fn, comm, color, key, newcomm, config) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclGetErrorString, ncclGetErrorString, ncclGetErrorString_fn, code) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclGetLastError, ncclGetLastError, ncclGetLastError_fn, comm) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclCommGetAsyncError, ncclCommGetAsyncError, ncclCommGetAsyncError_fn, comm, asyncError) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclCommCount, ncclCommCount, ncclCommCount_fn, comm, count) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclCommCuDevice, ncclCommCuDevice, ncclCommCuDevice_fn, comm, devid) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclCommUserRank, ncclCommUserRank, ncclCommUserRank_fn, comm, rank) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclMemAlloc, ncclMemAlloc, ncclMemAlloc_fn, ptr, size) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclMemFree, ncclMemFree, ncclMemFree_fn, ptr) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_mscclLoadAlgo, mscclLoadAlgo, mscclLoadAlgo_fn, mscclAlgoFilePath, mscclAlgoHandle, rank) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_mscclRunAlgo, mscclRunAlgo, mscclRunAlgo_fn, sendBuff, sendCounts, sDisPls, recvBuff, recvCounts, rDisPls, count, dataType, root, peer, op, mscclAlgoHandle, comm, stream) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_mscclUnloadAlgo, mscclUnloadAlgo, mscclUnloadAlgo_fn, mscclAlgoHandle) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclCommRegister, ncclCommRegister, ncclCommRegister_fn, comm, buff, size, handle) +RCCL_API_INFO_DEFINITION_V(ROCPROFILER_RCCL_TABLE_ID, ROCPROFILER_RCCL_API_ID_ncclCommDeregister, ncclCommDeregister, ncclCommDeregister_fn, comm, handle) + +#else +# error \ + "Do not compile this file directly. It is included by lib/rocprofiler-sdk/rccl/rccl.cpp" +#endif diff --git a/source/lib/rocprofiler-sdk/rccl/rccl.hpp b/source/lib/rocprofiler-sdk/rccl/rccl.hpp new file mode 100644 index 0000000000..3c3eeeb75f --- /dev/null +++ b/source/lib/rocprofiler-sdk/rccl/rccl.hpp @@ -0,0 +1,123 @@ +// MIT License +// +// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#pragma once + +#if !defined(ROCPROFILER_SDK_RCCL_HAS_API_TRACE) +# if defined __has_include +# if __has_include() +# define ROCPROFILER_SDK_RCCL_HAS_API_TRACE 1 +# else +# define ROCPROFILER_SDK_RCCL_HAS_API_TRACE 0 +# endif +# else +# define ROCPROFILER_SDK_RCCL_HAS_API_TRACE 0 +# endif +#endif + +#if ROCPROFILER_SDK_RCCL_HAS_API_TRACE > 0 +# include +#else +# include "lib/rocprofiler-sdk/rccl/details/api_trace.h" +#endif + +#include + +#include + +#include +#include + +namespace rocprofiler +{ +namespace rccl +{ +using rccl_api_func_table_t = ::rcclApiFuncTable; + +struct RcclAPITable +{ + rccl_api_func_table_t* rccl_api_table = nullptr; +}; + +using rccl_api_table_t = RcclAPITable; + +rccl_api_table_t& +get_table(); + +template +struct rccl_table_lookup; + +template +struct rccl_table_id_lookup; + +template +struct rccl_domain_info; + +template +struct rccl_api_info; + +template +struct rccl_api_impl : rccl_domain_info +{ + template + static auto set_data_args(DataArgsT&, Args... args); + + template + static auto exec(FuncT&&, Args&&... args); + + template + static RetT functor(Args... args); +}; + +template +const char* +name_by_id(uint32_t id); + +template +uint32_t +id_by_name(const char* name); + +template +std::vector +get_names(); + +template +std::vector +get_ids(); + +template +void +iterate_args(uint32_t id, + const rocprofiler_callback_tracing_rccl_api_data_t& data, + rocprofiler_callback_tracing_operation_args_cb_t callback, + int32_t max_deref, + void* user_data); + +template +void +copy_table(TableT* _orig, uint64_t _tbl_instance); + +template +void +update_table(TableT* _orig); +} // namespace rccl +} // namespace rocprofiler diff --git a/source/lib/rocprofiler-sdk/registration.cpp b/source/lib/rocprofiler-sdk/registration.cpp index 0fdec5f8cc..65d788938b 100644 --- a/source/lib/rocprofiler-sdk/registration.cpp +++ b/source/lib/rocprofiler-sdk/registration.cpp @@ -43,6 +43,7 @@ #include "lib/rocprofiler-sdk/page_migration/page_migration.hpp" #include "lib/rocprofiler-sdk/pc_sampling/code_object.hpp" #include "lib/rocprofiler-sdk/pc_sampling/service.hpp" +#include "lib/rocprofiler-sdk/rccl/rccl.hpp" #include #include @@ -846,6 +847,25 @@ rocprofiler_set_api_table(const char* name, rocprofiler::intercept_table::notify_intercept_table_registration( ROCPROFILER_MARKER_NAME_TABLE, lib_version, lib_instance, std::make_tuple(roctx_name)); } + else if(std::string_view{name} == "rccl") + { + // pass to rccl init + ROCP_ERROR_IF(num_tables > 1) + << "rocprofiler expected RCCL library to pass 1 API table, not " << num_tables; + + auto* rccl_api = static_cast(tables[0]); + + // any internal modifications to the rcclApiFuncTable need to be done before we make the + // copy or else those modifications will be lost when RCCL API tracing is enabled + // because the RCCL API tracing invokes the function pointers from the copy below + rocprofiler::rccl::copy_table(rccl_api, lib_instance); + + // install rocprofiler API wrappers + rocprofiler::rccl::update_table(rccl_api); + + rocprofiler::intercept_table::notify_intercept_table_registration( + ROCPROFILER_RCCL_TABLE, lib_version, lib_instance, std::make_tuple(rccl_api)); + } else { ROCP_ERROR << "rocprofiler does not accept API tables from " << name; diff --git a/tests/tools/json-tool.cpp b/tests/tools/json-tool.cpp index affb3228a1..a3d94dfda4 100644 --- a/tests/tools/json-tool.cpp +++ b/tests/tools/json-tool.cpp @@ -344,6 +344,23 @@ struct marker_api_callback_record_t } }; +struct rccl_api_callback_record_t +{ + uint64_t timestamp = 0; + rocprofiler_callback_tracing_record_t record = {}; + rocprofiler_callback_tracing_rccl_api_data_t payload = {}; + callback_arg_array_t args = {}; + + template + void save(ArchiveT& ar) const + { + ar(cereal::make_nvp("timestamp", timestamp)); + cereal::save(ar, record); + ar(cereal::make_nvp("payload", payload)); + serialize_args(ar, args); + } +}; + struct kernel_dispatch_callback_record_t { uint64_t timestamp = 0; @@ -466,6 +483,7 @@ auto hip_api_cb_records = std::deque{}; auto scratch_memory_cb_records = std::deque{}; auto kernel_dispatch_cb_records = std::deque{}; auto memory_copy_cb_records = std::deque{}; +auto rccl_api_cb_records = std::deque{}; int set_external_correlation_id(rocprofiler_thread_id_t thr_id, @@ -678,6 +696,19 @@ tool_tracing_callback(rocprofiler_callback_tracing_record_t record, auto _lk = std::unique_lock{_mutex}; memory_copy_cb_records.emplace_back(memory_copy_callback_record_t{ts, record, *data}); } + else if(record.kind == ROCPROFILER_CALLBACK_TRACING_RCCL_API) + { + auto* data = static_cast(record.payload); + auto args = callback_arg_array_t{}; + if(record.phase == ROCPROFILER_CALLBACK_PHASE_EXIT) + rocprofiler_iterate_callback_tracing_kind_operation_args( + record, save_args, record.phase, &args); + + static auto _mutex = std::mutex{}; + auto _lk = std::unique_lock{_mutex}; + rccl_api_cb_records.emplace_back( + rccl_api_callback_record_t{ts, record, *data, std::move(args)}); + } else { throw std::runtime_error{"unsupported callback kind"}; @@ -693,6 +724,7 @@ auto scratch_memory_records = std::deque{}; auto corr_id_retire_records = std::deque{}; +auto rccl_api_bf_records = std::deque{}; void tool_tracing_buffered(rocprofiler_context_id_t /*context*/, @@ -790,6 +822,13 @@ tool_tracing_buffered(rocprofiler_context_id_t /*context*/, corr_id_retire_records.emplace_back(*record); } + else if(header->kind == ROCPROFILER_BUFFER_TRACING_RCCL_API) + { + auto* record = + static_cast(header->payload); + + rccl_api_bf_records.emplace_back(*record); + } else { throw std::runtime_error{ @@ -869,11 +908,13 @@ rocprofiler_context_id_t hsa_api_callback_ctx = {}; rocprofiler_context_id_t hip_api_callback_ctx = {}; rocprofiler_context_id_t marker_api_callback_ctx = {}; rocprofiler_context_id_t code_object_ctx = {}; +rocprofiler_context_id_t rccl_api_callback_ctx = {}; rocprofiler_context_id_t hsa_api_buffered_ctx = {}; rocprofiler_context_id_t hip_api_buffered_ctx = {}; rocprofiler_context_id_t marker_api_buffered_ctx = {}; rocprofiler_context_id_t memory_copy_callback_ctx = {}; rocprofiler_context_id_t memory_copy_buffered_ctx = {}; +rocprofiler_context_id_t rccl_api_buffered_ctx = {}; rocprofiler_context_id_t counter_collection_ctx = {}; rocprofiler_context_id_t scratch_memory_ctx = {}; rocprofiler_context_id_t corr_id_retire_ctx = {}; @@ -890,6 +931,7 @@ rocprofiler_buffer_id_t page_migration_buffer = {}; rocprofiler_buffer_id_t counter_collection_buffer = {}; rocprofiler_buffer_id_t scratch_memory_buffer = {}; rocprofiler_buffer_id_t corr_id_retire_buffer = {}; +rocprofiler_buffer_id_t rccl_api_buffered_buffer = {}; auto contexts = std::unordered_map{ {"HSA_API_CALLBACK", &hsa_api_callback_ctx}, @@ -898,6 +940,7 @@ auto contexts = std::unordered_map{ {"CODE_OBJECT", &code_object_ctx}, {"KERNEL_DISPATCH_CALLBACK", &kernel_dispatch_callback_ctx}, {"MEMORY_COPY_CALLBACK", &memory_copy_callback_ctx}, + {"RCCL_API_CALLBACK", &rccl_api_callback_ctx}, {"HSA_API_BUFFERED", &hsa_api_buffered_ctx}, {"HIP_API_BUFFERED", &hip_api_buffered_ctx}, {"MARKER_API_BUFFERED", &marker_api_buffered_ctx}, @@ -907,17 +950,19 @@ auto contexts = std::unordered_map{ {"COUNTER_COLLECTION", &counter_collection_ctx}, {"SCRATCH_MEMORY", &scratch_memory_ctx}, {"CORRELATION_ID_RETIREMENT", &corr_id_retire_ctx}, + {"RCCL_API_BUFFERED", &rccl_api_buffered_ctx}, }; -auto buffers = std::array{&hsa_api_buffered_buffer, - &hip_api_buffered_buffer, - &marker_api_buffered_buffer, - &kernel_dispatch_buffer, - &memory_copy_buffer, - &scratch_memory_buffer, - &page_migration_buffer, - &counter_collection_buffer, - &corr_id_retire_buffer}; +auto buffers = std::array{&hsa_api_buffered_buffer, + &hip_api_buffered_buffer, + &marker_api_buffered_buffer, + &kernel_dispatch_buffer, + &memory_copy_buffer, + &scratch_memory_buffer, + &page_migration_buffer, + &counter_collection_buffer, + &corr_id_retire_buffer, + &rccl_api_buffered_buffer}; auto agents = std::vector{}; auto agents_map = std::unordered_map{}; @@ -1056,6 +1101,18 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data) nullptr), "scratch memory tracing service configure"); + { + auto _status = + rocprofiler_configure_callback_tracing_service(rccl_api_callback_ctx, + ROCPROFILER_CALLBACK_TRACING_RCCL_API, + nullptr, + 0, + tool_tracing_callback, + nullptr); + if(_status != ROCPROFILER_STATUS_ERROR_NOT_IMPLEMENTED) + ROCPROFILER_CALL(_status, "rccl api callback tracing service configure"); + } + constexpr auto buffer_size = 8192; constexpr auto watermark = 7936; @@ -1140,6 +1197,15 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data) &counter_collection_buffer), "buffer creation"); + ROCPROFILER_CALL(rocprofiler_create_buffer(rccl_api_buffered_ctx, + buffer_size, + watermark, + ROCPROFILER_BUFFER_POLICY_LOSSLESS, + tool_tracing_buffered, + tool_data, + &rccl_api_buffered_buffer), + "buffer creation"); + for(auto itr : {ROCPROFILER_BUFFER_TRACING_HSA_CORE_API, ROCPROFILER_BUFFER_TRACING_HSA_AMD_EXT_API, ROCPROFILER_BUFFER_TRACING_HSA_IMAGE_EXT_API, @@ -1231,6 +1297,17 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data) corr_id_retire_buffer), "buffer tracing service for memory copy configure"); + { + auto _status = + rocprofiler_configure_buffer_tracing_service(rccl_api_buffered_ctx, + ROCPROFILER_BUFFER_TRACING_RCCL_API, + nullptr, + 0, + rccl_api_buffered_buffer); + if(_status != ROCPROFILER_STATUS_ERROR_NOT_IMPLEMENTED) + ROCPROFILER_CALL(_status, "buffer tracing service configure"); + } + ROCPROFILER_CALL( rocprofiler_configure_buffered_dispatch_profile_counting_service( counter_collection_ctx, counter_collection_buffer, dispatch_callback, nullptr), @@ -1377,6 +1454,7 @@ tool_fini(void* tool_data) << ", scratch_memory_callback_records=" << scratch_memory_cb_records.size() << ", kernel_dispatch_callback_records=" << kernel_dispatch_cb_records.size() << ", memory_copy_callback_records=" << memory_copy_cb_records.size() + << ", rccl_api_callback_records=" << rccl_api_cb_records.size() << ", kernel_dispatch_bf_records=" << kernel_dispatch_bf_records.size() << ", memory_copy_bf_records=" << memory_copy_bf_records.size() << ", scratch_memory_records=" << scratch_memory_records.size() @@ -1385,6 +1463,7 @@ tool_fini(void* tool_data) << ", hip_api_bf_records=" << hip_api_bf_records.size() << ", marker_api_bf_records=" << marker_api_bf_records.size() << ", corr_id_retire_records=" << corr_id_retire_records.size() + << ", rccl_api_bf_records=" << rccl_api_bf_records.size() << ", counter_collection_value_records=" << counter_collection_bf_records.size() << "...\n" << std::flush; @@ -1474,6 +1553,7 @@ write_json(call_stack_t* _call_stack) json_ar(cereal::make_nvp("hsa_api_traces", hsa_api_cb_records)); json_ar(cereal::make_nvp("hip_api_traces", hip_api_cb_records)); json_ar(cereal::make_nvp("marker_api_traces", marker_api_cb_records)); + json_ar(cereal::make_nvp("rccl_api_traces", rccl_api_cb_records)); json_ar(cereal::make_nvp("scratch_memory_traces", scratch_memory_cb_records)); json_ar(cereal::make_nvp("kernel_dispatch", kernel_dispatch_cb_records)); json_ar(cereal::make_nvp("memory_copies", memory_copy_cb_records)); @@ -1497,6 +1577,7 @@ write_json(call_stack_t* _call_stack) json_ar(cereal::make_nvp("hsa_api_traces", hsa_api_bf_records)); json_ar(cereal::make_nvp("hip_api_traces", hip_api_bf_records)); json_ar(cereal::make_nvp("marker_api_traces", marker_api_bf_records)); + json_ar(cereal::make_nvp("rccl_api_traces", rccl_api_bf_records)); json_ar(cereal::make_nvp("retired_correlation_ids", corr_id_retire_records)); json_ar(cereal::make_nvp("counter_collection", counter_collection_bf_records)); } catch(std::exception& e) @@ -1565,6 +1646,8 @@ write_perfetto() tids.emplace(itr.thread_id); for(auto itr : marker_api_bf_records) tids.emplace(itr.thread_id); + for(auto itr : rccl_api_bf_records) + tids.emplace(itr.thread_id); for(auto itr : memory_copy_bf_records) { @@ -1743,6 +1826,45 @@ write_perfetto() itr.end_timestamp); } + for(auto itr : rccl_api_bf_records) + { + auto name = buffer_names.at(itr.kind, itr.operation); + auto& track = thread_tracks.at(itr.thread_id); + + auto _args = callback_arg_array_t{}; + auto ritr = std::find_if( + rccl_api_cb_records.begin(), rccl_api_cb_records.end(), [&itr](const auto& citr) { + return (citr.record.correlation_id.internal == itr.correlation_id.internal && + !citr.args.empty()); + }); + if(ritr != rccl_api_cb_records.end()) _args = ritr->args; + + TRACE_EVENT_BEGIN(sdk::perfetto_category::name, + ::perfetto::StaticString(name.data()), + track, + itr.start_timestamp, + ::perfetto::Flow::ProcessScoped(itr.correlation_id.internal), + "begin_ns", + itr.start_timestamp, + "tid", + itr.thread_id, + "kind", + itr.kind, + "operation", + itr.operation, + "corr_id", + itr.correlation_id.internal, + [&](::perfetto::EventContext ctx) { + for(const auto& aitr : _args) + sdk::add_perfetto_annotation(ctx, aitr.first, aitr.second); + }); + TRACE_EVENT_END(sdk::perfetto_category::name, + track, + itr.end_timestamp, + "end_ns", + itr.end_timestamp); + } + for(auto itr : memory_copy_bf_records) { auto name = buffer_names.at(itr.kind, itr.operation);