v0.0.3: MPI wrappers w/o full MPI support + setup-env.sh (#15)

* v0.0.3: MPI wrappers w/o full MPI support + setup-env.sh

- bumped to v0.0.3
- enabled gotcha wrapping of MPI functions w/o enabling MPI
- added setup-env.sh script
- minor updates to testing
- Update timemory submodule
- fixes tim::component::configure_mpip undefined symbol
- Script updates

[ROCm/rocprofiler-systems commit: c56b49a0bd]
Этот коммит содержится в:
Jonathan R. Madsen
2021-10-01 16:46:03 -05:00
коммит произвёл GitHub
родитель 2cc8005680
Коммит cdd2707058
15 изменённых файлов: 188 добавлений и 36 удалений
+27 -12
Просмотреть файл
@@ -9,16 +9,23 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_BINARY_DIR AND CMAKE_CURRENT_SOURCE_DIR STREQ
message(AUTHOR_WARNING "In-source build")
endif()
if(NOT UNIX OR APPLE)
message(
AUTHOR_WARNING
"hosttrace only supports Linux. Configure and/or build is likely to fail")
endif()
project(
hosttrace
LANGUAGES C CXX
VERSION 0.0.2)
VERSION 0.0.3)
message(
STATUS
"[${PROJECT_NAME}] version ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}"
)
set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH})
set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake ${PROJECT_SOURCE_DIR}/cmake/Modules
${CMAKE_MODULE_PATH})
set(BUILD_SHARED_LIBS
ON
CACHE BOOL "Build shared libraries")
@@ -51,14 +58,13 @@ add_option(HOSTTRACE_USE_MPI "Enable MPI support" OFF)
add_option(HOSTTRACE_CUSTOM_DATA_SOURCE "Enable custom data source" OFF)
add_option(HOSTTRACE_USE_ROCTRACER "Enable roctracer support" ON)
add_option(HOSTTRACE_BUILD_DYNINST "Build dyninst from submodule" OFF)
add_option(HOSTTRACE_USE_MPI_HEADERS
"Enable wrapping MPI functions w/o enabling MPI dependency" OFF)
# ensure synced
set(TIMEMORY_USE_MPI
${HOSTTRACE_USE_MPI}
CACHE BOOL "Enable MPI support" FORCE)
set(TRANSPOSE_USE_MPI
${HOSTTRACE_USE_MPI}
CACHE BOOL "Enable MPI support" FORCE)
include(Packages) # finds third-party libraries
@@ -98,19 +104,20 @@ target_link_libraries(
PRIVATE hosttrace::hosttrace-threading
hosttrace::hosttrace-compile-options
hosttrace::hosttrace-roctracer
hosttrace::hosttrace-mpi
$<BUILD_INTERFACE:timemory::timemory-headers>
$<BUILD_INTERFACE:timemory::timemory-gotcha>
$<BUILD_INTERFACE:timemory::timemory-cxx-shared>
$<IF:$<BOOL:${HOSTTRACE_USE_SANITIZER}>,hosttrace::hosttrace-sanitizer,>)
if(DYNINST_API_RT)
get_filename_component(DYNINST_API_RT_DIR "${DYNINST_API_RT}" DIRECTORY)
if(HOSTTRACE_DYNINST_API_RT)
get_filename_component(HOSTTRACE_DYNINST_API_RT_DIR "${HOSTTRACE_DYNINST_API_RT}"
DIRECTORY)
endif()
set_target_properties(
hosttrace-library
PROPERTIES OUTPUT_NAME hosttrace
INSTALL_RPATH "\$ORIGIN:\$ORIGIN/dyninst-tpls/libs:${DYNINST_API_RT_DIR}")
hosttrace-library PROPERTIES OUTPUT_NAME hosttrace
INSTALL_RPATH "\$ORIGIN:\$ORIGIN/dyninst-tpls/libs")
install(
TARGETS hosttrace-library
@@ -143,7 +150,7 @@ set_target_properties(
OUTPUT_NAME hosttrace
INSTALL_RPATH_USE_LINK_PATH ON
INSTALL_RPATH
"\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}:\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}/dyninst-tpls/lib:${DYNINST_API_RT_DIR}"
"\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}:\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}/dyninst-tpls/lib"
)
install(
@@ -160,6 +167,9 @@ add_dependencies(hosttrace-exe timemory-avail)
#
# ------------------------------------------------------------------------------#
configure_file(${PROJECT_SOURCE_DIR}/scripts/setup-env.sh.in
${PROJECT_BINARY_DIR}/scripts/setup-env.sh @ONLY)
install(
PROGRAMS ${PROJECT_SOURCE_DIR}/scripts/hosttrace-merge.jl
DESTINATION ${CMAKE_INSTALL_BINDIR}
@@ -167,7 +177,12 @@ install(
install(
FILES ${PROJECT_SOURCE_DIR}/roctrace.cfg
DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}
DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME}
OPTIONAL)
install(
FILES ${PROJECT_BINARY_DIR}/scripts/setup-env.sh
DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME}
OPTIONAL)
# ------------------------------------------------------------------------------#
+41
Просмотреть файл
@@ -0,0 +1,41 @@
# ------------------------------------------------------------------------------#
#
# Finds headers for MPI
#
# ------------------------------------------------------------------------------#
include(FindPackageHandleStandardArgs)
find_path(
MPI_HEADERS_INCLUDE_DIR
NAMES mpi.h
PATH_SUFFIXES include include/mpich include/openmpi mpich openmpi
HINTS ${MPI_ROOT_DIR}
PATHS ${MPI_ROOT_DIR})
if(NOT MPI_HEADERS_INCLUDE_DIR)
find_package(MPI QUIET)
if(MPI_C_INCLUDE_DIRS)
set(MPI_HEADERS_INCLUDE_DIR
${MPI_C_INCLUDE_DIRS}
CACHE PATH "Include directory for MPI" FORCE)
elseif(MPI_CXX_INCLUDE_DIRS)
set(MPI_HEADERS_INCLUDE_DIR
${MPI_CXX_INCLUDE_DIRS}
CACHE PATH "Include directory for MPI" FORCE)
endif()
endif()
if(MPI_HEADERS_INCLUDE_DIR)
set(MPI_HEADERS_INCLUDE_DIRS ${MPI_HEADERS_INCLUDE_DIR})
endif()
mark_as_advanced(MPI_HEADERS_INCLUDE_DIR)
find_package_handle_standard_args(MPI-Headers REQUIRED_VARS MPI_HEADERS_INCLUDE_DIR)
if(MPI-Headers_FOUND)
add_library(MPI::MPI_HEADERS IMPORTED INTERFACE)
target_include_directories(
MPI::MPI_HEADERS INTERFACE $<$<COMPILE_LANGUAGE:C>:${MPI_HEADERS_INCLUDE_DIR}>
$<$<COMPILE_LANGUAGE:CXX>:${MPI_HEADERS_INCLUDE_DIR}>)
endif()
+21 -4
Просмотреть файл
@@ -14,6 +14,7 @@ add_interface_library(
hosttrace-dyninst
"Provides flags and libraries for Dyninst (dynamic instrumentation)")
add_interface_library(hosttrace-roctracer "Provides flags and libraries for roctracer")
add_interface_library(hosttrace-mpi "Provides MPI or MPI headers")
# include threading because of rooflines
target_link_libraries(hosttrace-headers INTERFACE hosttrace-threading)
@@ -56,6 +57,22 @@ if(HOSTTRACE_USE_ROCTRACER)
set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${roctracer_LIBRARY_DIRS}")
endif()
# ----------------------------------------------------------------------------------------#
#
# MPI
#
# ----------------------------------------------------------------------------------------#
if(HOSTTRACE_USE_MPI)
find_package(MPI ${hosttrace_FIND_QUIETLY} REQUIRED)
target_link_libraries(hosttrace-mpi INTERFACE MPI::MPI_C MPI::MPI_CXX)
elseif(HOSTTRACE_USE_MPI_HEADERS)
find_package(MPI-Headers ${hosttrace_FIND_QUIETLY} REQUIRED)
target_compile_definitions(hosttrace-mpi INTERFACE TIMEMORY_USE_MPI_HEADERS
HOSTTRACE_USE_MPI_HEADERS)
target_link_libraries(hosttrace-mpi INTERFACE MPI::MPI_HEADERS)
endif()
# ----------------------------------------------------------------------------------------#
#
# Dyninst
@@ -153,7 +170,7 @@ else()
# useful for defining the location of the runtime API
find_library(
DYNINST_API_RT dyninstAPI_RT
HOSTTRACE_DYNINST_API_RT dyninstAPI_RT
HINTS ${Dyninst_ROOT_DIR} ${Dyninst_DIR}
PATHS ${Dyninst_ROOT_DIR} ${Dyninst_DIR}
PATH_SUFFIXES lib)
@@ -174,9 +191,9 @@ else()
PATH_SUFFIXES include)
endif()
if(DYNINST_API_RT)
target_compile_definitions(hosttrace-dyninst
INTERFACE DYNINST_API_RT="${DYNINST_API_RT}")
if(HOSTTRACE_DYNINST_API_RT)
target_compile_definitions(
hosttrace-dyninst INTERFACE DYNINST_API_RT="${HOSTTRACE_DYNINST_API_RT}")
endif()
if(Boost_DIR)
+4 -2
Просмотреть файл
@@ -12,14 +12,16 @@ ENV DEBIAN_FRONTEND noninteractive
WORKDIR /tmp
SHELL [ "/bin/bash", "-c" ]
ARG EXTRA_PACKAGES=""
RUN apt-get update && \
apt-get dist-upgrade -y && \
apt-get install -y build-essential cmake libnuma-dev wget gnupg2 m4 && \
apt-get install -y build-essential cmake libnuma-dev wget gnupg2 m4 bash-completion git-core && \
wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - && \
echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/debian/ ubuntu main' | tee /etc/apt/sources.list.d/rocm.list && \
apt-get update && \
apt-get dist-upgrade -y && \
apt-get install -y rocm-dev rocm-utils roctracer-dev rocprofiler-dev hip-base hsa-amd-aqlprofile hsa-rocr-dev hsakmt-roct-dev
apt-get install -y rocm-dev rocm-utils roctracer-dev rocprofiler-dev hip-base hsa-amd-aqlprofile hsa-rocr-dev hsakmt-roct-dev ${EXTRA_PACKAGES}
ENV LC_ALL C.UTF-8
WORKDIR /home
+20 -4
Просмотреть файл
@@ -84,14 +84,14 @@ transpose_a(int* in, int* out, int M, int N)
}
void
run(int argc, char** argv)
run(int rank, int argc, char** argv)
{
(void) argc;
(void) argv;
unsigned int M = 4960;
unsigned int N = 4960;
std::cout << "M: " << M << " N: " << N << std::endl;
std::cout << "[" << rank << "] M: " << M << " N: " << N << std::endl;
size_t size = sizeof(int) * M * N;
int* matrix = (int*) malloc(size);
for(size_t i = 0; i < M * N; i++)
@@ -130,7 +130,7 @@ run(int argc, char** argv)
double time =
std::chrono::duration_cast<std::chrono::duration<double>>(t2 - t1).count();
float GB = (float) size * times * 2 / (1 << 30);
std::cout << "Runtime of transpose is " << time << " sec\n"
std::cout << "[" << rank << "] Runtime of transpose is " << time << " sec\n"
<< "The average performance of transpose is " << GB / time << " GBytes/sec"
<< std::endl;
@@ -151,6 +151,21 @@ run(int argc, char** argv)
#if defined(USE_MPI)
# include <mpi.h>
void
do_a2a(int rank)
{
// Define my value
int values[3];
for(int i = 0; i < 3; ++i)
values[i] = rank * 300 + i * 100;
printf("Process %d, values = %d, %d, %d.\n", rank, values[0], values[1], values[2]);
int buffer_recv[3];
MPI_Alltoall(&values, 1, MPI_INT, buffer_recv, 1, MPI_INT, MPI_COMM_WORLD);
printf("Values collected on process %d: %d, %d, %d.\n", rank, buffer_recv[0],
buffer_recv[1], buffer_recv[2]);
}
#endif
int
@@ -161,9 +176,10 @@ main(int argc, char** argv)
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
#endif
if(rank == 0) run(argc, argv);
if(rank == 0) run(rank, argc, argv);
#if defined(USE_MPI)
MPI_Barrier(MPI_COMM_WORLD);
do_a2a(rank);
MPI_Finalize();
#endif
return 0;
Submodule projects/rocprofiler-systems/external/timemory updated: 9f27fafd2e...11183bbdd7
+7 -1
Просмотреть файл
@@ -395,7 +395,13 @@ struct module_function
module = modname;
function = fname;
signature = get_func_file_line_info(mod, proc);
assert(proc->isInstrumentable() == true);
if(!proc->isInstrumentable())
{
verbprintf(0,
"Warning! module function generated for un-instrumentable "
"function: %s [%s]\n",
function.c_str(), module.c_str());
}
std::pair<address_t, address_t> _range{};
if(proc->getAddressRange(_range.first, _range.second))
address_range = _range.second - _range.first;
+3
Просмотреть файл
@@ -115,6 +115,9 @@ struct mpi_gotcha : comp::base<mpi_gotcha, void>
// this will get called right after MPI_Init and MPI_Init_thread with the return value
void audit(const gotcha_data_t& _data, audit::outgoing, int _retval);
// this will get called right before MPI_Finalize
void audit(const gotcha_data_t& _data, audit::incoming);
};
// timemory api struct
+14
Просмотреть файл
@@ -0,0 +1,14 @@
#!/bin/bash
VERSION=0.0.3
ROCM_VERSION=4.3.0
STANDARD_ARGS="-DCPACK_GENERATOR=STGZ -DCMAKE_BUILD_TYPE=Release -DHOSTTRACE_BUILD_DYNINST=ON -DTIMEMORY_BUILD_PORTABLE=ON"
cmake -B build-release/core ${STANDARD_ARGS} -DDYNINST_BUILD_{TBB,BOOST,ELFUTILS,LIBIBERTY}=ON -DDYNINST_USE_OpenMP=OFF -DHOSTTRACE_USE_MPI_HEADERS=ON -DHOSTTRACE_USE_ROCTRACER=OFF .
cmake --build build-release/core --target package --parallel 8
cp build-release/core/hosttrace-${VERSION}-Linux.sh build-release/hosttrace-${VERSION}-Linux.sh
cmake -B build-release/rocm-mpi ${STANDARD_ARGS} -DDYNINST_BUILD_{TBB,BOOST,ELFUTILS,LIBIBERTY}=ON -DDYNINST_USE_OpenMP=ON -DHOSTTRACE_USE_MPI_HEADERS=ON -DHOSTTRACE_USE_ROCTRACER=ON .
cmake --build build-release/rocm-mpi --target package --parallel 8
cp build-release/rocm-mpi/hosttrace-${VERSION}-Linux.sh build-release/hosttrace-${VERSION}-Linux-ROCm-${ROCM_VERSION}.sh
+14
Просмотреть файл
@@ -0,0 +1,14 @@
#!/usr/bin/env bash
BASEDIR=$(dirname ${BASH_SOURCE[0]})
BASEDIR=$(cd ${BASEDIR}/../.. && pwd)
if [ ! -d "${BASEDIR}" ]; then
echo "${BASEDIR} does not exist"
return 1
fi
export PATH=${BASEDIR}/bin:${PATH}
export LD_LIBRARY_PATH=${BASEDIR}/@CMAKE_INSTALL_LIBDIR@:${LD_LIBRARY_PATH}
return 0
+10 -7
Просмотреть файл
@@ -730,12 +730,14 @@ main(int argc, char** argv)
std::set<std::string> module_names;
auto _add_overlapping = [](module_t* mitr, procedure_t* pitr) {
if(!pitr->isInstrumentable()) return;
std::vector<procedure_t*> _overlapping{};
if(pitr->findOverlapping(_overlapping))
{
overlapping_module_functions.insert(module_function{ mitr, pitr });
for(auto oitr : _overlapping)
{
if(!oitr->isInstrumentable()) continue;
overlapping_module_functions.insert(
module_function{ oitr->getModule(), oitr });
}
@@ -752,7 +754,8 @@ main(int argc, char** argv)
{
for(auto* pitr : *procedures)
{
auto _modfn = module_function(itr, pitr);
if(!pitr->isInstrumentable()) continue;
auto _modfn = module_function{ itr, pitr };
module_names.insert(_modfn.module);
available_module_functions.insert(std::move(_modfn));
_add_overlapping(itr, pitr);
@@ -771,9 +774,9 @@ main(int argc, char** argv)
for(auto* itr : *app_functions)
{
module_t* mod = itr->getModule();
if(mod)
if(mod && itr->isInstrumentable())
{
auto _modfn = module_function(mod, itr);
auto _modfn = module_function{ mod, itr };
module_names.insert(_modfn.module);
available_module_functions.insert(std::move(_modfn));
_add_overlapping(mod, itr);
@@ -1332,11 +1335,11 @@ main(int argc, char** argv)
else
itr->getModuleName(modname, FUNCNAMELEN);
if(itr == main_func)
if(itr == main_func && main_func->isInstrumentable())
{
hash_ids.emplace_back(std::hash<string_t>()(main_sign.get()),
main_sign.get());
auto main_mf = module_function(modname, fname, main_sign, itr);
auto main_mf = module_function{ modname, fname, main_sign, itr };
available_module_functions.insert(main_mf);
instrumented_module_functions.insert(main_mf);
continue;
@@ -1460,8 +1463,8 @@ main(int argc, char** argv)
}
hash_ids.emplace_back(std::hash<string_t>()(name.get()), name.get());
available_module_functions.insert(module_function(mod, itr));
instrumented_module_functions.insert(module_function(mod, itr));
available_module_functions.insert(module_function{ mod, itr });
instrumented_module_functions.insert(module_function{ mod, itr });
auto _f = [=]() {
verbprintf(1, "Instrumenting |> [ %s ] -> [ %s ]\n", modname,
+11
Просмотреть файл
@@ -91,6 +91,17 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::outgoing, int _retval)
}
}
void
mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming)
{
HOSTTRACE_DEBUG("[%s] %s()\n", __FUNCTION__, _data.tool_id.c_str());
if(mpip_index != std::numeric_limits<uint64_t>::max())
comp::deactivate_mpip<tim::component_tuple<hosttrace_component>, hosttrace>(
mpip_index);
hosttrace_pop_trace("MPI_Finalize()");
hosttrace_trace_finalize();
}
void
hosttrace_component::start()
{
+3
Просмотреть файл
@@ -83,6 +83,9 @@ setup_gotchas()
mpi_gotcha_t::template configure<0, int, int*, char***>("MPI_Init");
mpi_gotcha_t::template configure<1, int, int*, char***, int, int*>(
"MPI_Init_thread");
#if defined(HOSTTRACE_USE_MPI_HEADERS)
mpi_gotcha_t::template configure<3, int>("MPI_Finalize");
#endif
};
}
+12 -5
Просмотреть файл
@@ -1,18 +1,25 @@
if(NOT DYNINST_API_RT_DIR AND DYNINST_API_RT)
get_filename_component(DYNINST_API_RT_DIR "${DYNINST_API_RT}" DIRECTORY)
if(NOT HOSTTRACE_DYNINST_API_RT_DIR AND HOSTTRACE_DYNINST_API_RT)
get_filename_component(HOSTTRACE_DYNINST_API_RT_DIR "${HOSTTRACE_DYNINST_API_RT}"
DIRECTORY)
endif()
if(HOSTTRACE_BUILD_DYNINST)
set(DYNINST_API_RT_DIR
set(HOSTTRACE_DYNINST_API_RT_DIR
"${PROJECT_BINARY_DIR}/external/dyninst/dyninstAPI_RT:${PROJECT_BINARY_DIR}/external/dyninst/dyninstAPI"
)
endif()
set(_test_environment
"HOSTTRACE_USE_PERFETTO=ON"
"LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}:${DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}")
"HOSTTRACE_USE_TIMEMORY=ON"
"LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}:${HOSTTRACE_DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}"
)
if(TARGET transpose)
if(TRANSPOSE_USE_MPI)
set(COMMAND_PREFIX ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 2)
endif()
add_test(
NAME transpose-binary-rewrite
COMMAND
@@ -22,7 +29,7 @@ if(TARGET transpose)
add_test(
NAME transpose-binary-rewrite-run
COMMAND $<TARGET_FILE_DIR:transpose>/transpose.inst
COMMAND ${COMMAND_PREFIX} $<TARGET_FILE_DIR:transpose>/transpose.inst
WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
add_test(