From c56b49a0bdf4faacd539742078e2c7b36b78f726 Mon Sep 17 00:00:00 2001 From: "Jonathan R. Madsen" Date: Fri, 1 Oct 2021 16:46:03 -0500 Subject: [PATCH] v0.0.3: MPI wrappers w/o full MPI support + setup-env.sh (#15) * v0.0.3: MPI wrappers w/o full MPI support + setup-env.sh - bumped to v0.0.3 - enabled gotcha wrapping of MPI functions w/o enabling MPI - added setup-env.sh script - minor updates to testing - Update timemory submodule - fixes tim::component::configure_mpip undefined symbol - Script updates --- CMakeLists.txt | 39 +++++++++++++++-------- cmake/Modules/FindMPI-Headers.cmake | 41 +++++++++++++++++++++++++ cmake/{ => Modules}/Findroctracer.cmake | 0 cmake/Packages.cmake | 25 ++++++++++++--- docker/Dockerfile | 6 ++-- examples/transpose/transpose.cpp | 24 ++++++++++++--- external/timemory | 2 +- include/hosttrace.hpp | 8 ++++- include/library.hpp | 3 ++ scripts/build-release.sh | 14 +++++++++ scripts/setup-env.sh.in | 14 +++++++++ src/hosttrace.cpp | 17 +++++----- src/libmisc.cpp | 11 +++++++ src/library.cpp | 3 ++ tests/CMakeLists.txt | 17 +++++++--- 15 files changed, 188 insertions(+), 36 deletions(-) create mode 100644 cmake/Modules/FindMPI-Headers.cmake rename cmake/{ => Modules}/Findroctracer.cmake (100%) create mode 100755 scripts/build-release.sh create mode 100644 scripts/setup-env.sh.in diff --git a/CMakeLists.txt b/CMakeLists.txt index cc1732b22c..7ea87ca71c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,16 +9,23 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_BINARY_DIR AND CMAKE_CURRENT_SOURCE_DIR STREQ message(AUTHOR_WARNING "In-source build") endif() +if(NOT UNIX OR APPLE) + message( + AUTHOR_WARNING + "hosttrace only supports Linux. Configure and/or build is likely to fail") +endif() + project( hosttrace LANGUAGES C CXX - VERSION 0.0.2) + VERSION 0.0.3) message( STATUS "[${PROJECT_NAME}] version ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}" ) -set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH}) +set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake ${PROJECT_SOURCE_DIR}/cmake/Modules + ${CMAKE_MODULE_PATH}) set(BUILD_SHARED_LIBS ON CACHE BOOL "Build shared libraries") @@ -51,14 +58,13 @@ add_option(HOSTTRACE_USE_MPI "Enable MPI support" OFF) add_option(HOSTTRACE_CUSTOM_DATA_SOURCE "Enable custom data source" OFF) add_option(HOSTTRACE_USE_ROCTRACER "Enable roctracer support" ON) add_option(HOSTTRACE_BUILD_DYNINST "Build dyninst from submodule" OFF) +add_option(HOSTTRACE_USE_MPI_HEADERS + "Enable wrapping MPI functions w/o enabling MPI dependency" OFF) # ensure synced set(TIMEMORY_USE_MPI ${HOSTTRACE_USE_MPI} CACHE BOOL "Enable MPI support" FORCE) -set(TRANSPOSE_USE_MPI - ${HOSTTRACE_USE_MPI} - CACHE BOOL "Enable MPI support" FORCE) include(Packages) # finds third-party libraries @@ -98,19 +104,20 @@ target_link_libraries( PRIVATE hosttrace::hosttrace-threading hosttrace::hosttrace-compile-options hosttrace::hosttrace-roctracer + hosttrace::hosttrace-mpi $ $ $ $,hosttrace::hosttrace-sanitizer,>) -if(DYNINST_API_RT) - get_filename_component(DYNINST_API_RT_DIR "${DYNINST_API_RT}" DIRECTORY) +if(HOSTTRACE_DYNINST_API_RT) + get_filename_component(HOSTTRACE_DYNINST_API_RT_DIR "${HOSTTRACE_DYNINST_API_RT}" + DIRECTORY) endif() set_target_properties( - hosttrace-library - PROPERTIES OUTPUT_NAME hosttrace - INSTALL_RPATH "\$ORIGIN:\$ORIGIN/dyninst-tpls/libs:${DYNINST_API_RT_DIR}") + hosttrace-library PROPERTIES OUTPUT_NAME hosttrace + INSTALL_RPATH "\$ORIGIN:\$ORIGIN/dyninst-tpls/libs") install( TARGETS hosttrace-library @@ -143,7 +150,7 @@ set_target_properties( OUTPUT_NAME hosttrace INSTALL_RPATH_USE_LINK_PATH ON INSTALL_RPATH - "\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}:\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}/dyninst-tpls/lib:${DYNINST_API_RT_DIR}" + "\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}:\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}/dyninst-tpls/lib" ) install( @@ -160,6 +167,9 @@ add_dependencies(hosttrace-exe timemory-avail) # # ------------------------------------------------------------------------------# +configure_file(${PROJECT_SOURCE_DIR}/scripts/setup-env.sh.in + ${PROJECT_BINARY_DIR}/scripts/setup-env.sh @ONLY) + install( PROGRAMS ${PROJECT_SOURCE_DIR}/scripts/hosttrace-merge.jl DESTINATION ${CMAKE_INSTALL_BINDIR} @@ -167,7 +177,12 @@ install( install( FILES ${PROJECT_SOURCE_DIR}/roctrace.cfg - DESTINATION ${CMAKE_INSTALL_DATAROOTDIR} + DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME} + OPTIONAL) + +install( + FILES ${PROJECT_BINARY_DIR}/scripts/setup-env.sh + DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME} OPTIONAL) # ------------------------------------------------------------------------------# diff --git a/cmake/Modules/FindMPI-Headers.cmake b/cmake/Modules/FindMPI-Headers.cmake new file mode 100644 index 0000000000..87823dad00 --- /dev/null +++ b/cmake/Modules/FindMPI-Headers.cmake @@ -0,0 +1,41 @@ +# ------------------------------------------------------------------------------# +# +# Finds headers for MPI +# +# ------------------------------------------------------------------------------# + +include(FindPackageHandleStandardArgs) + +find_path( + MPI_HEADERS_INCLUDE_DIR + NAMES mpi.h + PATH_SUFFIXES include include/mpich include/openmpi mpich openmpi + HINTS ${MPI_ROOT_DIR} + PATHS ${MPI_ROOT_DIR}) + +if(NOT MPI_HEADERS_INCLUDE_DIR) + find_package(MPI QUIET) + if(MPI_C_INCLUDE_DIRS) + set(MPI_HEADERS_INCLUDE_DIR + ${MPI_C_INCLUDE_DIRS} + CACHE PATH "Include directory for MPI" FORCE) + elseif(MPI_CXX_INCLUDE_DIRS) + set(MPI_HEADERS_INCLUDE_DIR + ${MPI_CXX_INCLUDE_DIRS} + CACHE PATH "Include directory for MPI" FORCE) + endif() +endif() + +if(MPI_HEADERS_INCLUDE_DIR) + set(MPI_HEADERS_INCLUDE_DIRS ${MPI_HEADERS_INCLUDE_DIR}) +endif() + +mark_as_advanced(MPI_HEADERS_INCLUDE_DIR) +find_package_handle_standard_args(MPI-Headers REQUIRED_VARS MPI_HEADERS_INCLUDE_DIR) + +if(MPI-Headers_FOUND) + add_library(MPI::MPI_HEADERS IMPORTED INTERFACE) + target_include_directories( + MPI::MPI_HEADERS INTERFACE $<$:${MPI_HEADERS_INCLUDE_DIR}> + $<$:${MPI_HEADERS_INCLUDE_DIR}>) +endif() diff --git a/cmake/Findroctracer.cmake b/cmake/Modules/Findroctracer.cmake similarity index 100% rename from cmake/Findroctracer.cmake rename to cmake/Modules/Findroctracer.cmake diff --git a/cmake/Packages.cmake b/cmake/Packages.cmake index 853fb107b4..0ba6b6c340 100644 --- a/cmake/Packages.cmake +++ b/cmake/Packages.cmake @@ -14,6 +14,7 @@ add_interface_library( hosttrace-dyninst "Provides flags and libraries for Dyninst (dynamic instrumentation)") add_interface_library(hosttrace-roctracer "Provides flags and libraries for roctracer") +add_interface_library(hosttrace-mpi "Provides MPI or MPI headers") # include threading because of rooflines target_link_libraries(hosttrace-headers INTERFACE hosttrace-threading) @@ -56,6 +57,22 @@ if(HOSTTRACE_USE_ROCTRACER) set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${roctracer_LIBRARY_DIRS}") endif() +# ----------------------------------------------------------------------------------------# +# +# MPI +# +# ----------------------------------------------------------------------------------------# + +if(HOSTTRACE_USE_MPI) + find_package(MPI ${hosttrace_FIND_QUIETLY} REQUIRED) + target_link_libraries(hosttrace-mpi INTERFACE MPI::MPI_C MPI::MPI_CXX) +elseif(HOSTTRACE_USE_MPI_HEADERS) + find_package(MPI-Headers ${hosttrace_FIND_QUIETLY} REQUIRED) + target_compile_definitions(hosttrace-mpi INTERFACE TIMEMORY_USE_MPI_HEADERS + HOSTTRACE_USE_MPI_HEADERS) + target_link_libraries(hosttrace-mpi INTERFACE MPI::MPI_HEADERS) +endif() + # ----------------------------------------------------------------------------------------# # # Dyninst @@ -153,7 +170,7 @@ else() # useful for defining the location of the runtime API find_library( - DYNINST_API_RT dyninstAPI_RT + HOSTTRACE_DYNINST_API_RT dyninstAPI_RT HINTS ${Dyninst_ROOT_DIR} ${Dyninst_DIR} PATHS ${Dyninst_ROOT_DIR} ${Dyninst_DIR} PATH_SUFFIXES lib) @@ -174,9 +191,9 @@ else() PATH_SUFFIXES include) endif() - if(DYNINST_API_RT) - target_compile_definitions(hosttrace-dyninst - INTERFACE DYNINST_API_RT="${DYNINST_API_RT}") + if(HOSTTRACE_DYNINST_API_RT) + target_compile_definitions( + hosttrace-dyninst INTERFACE DYNINST_API_RT="${HOSTTRACE_DYNINST_API_RT}") endif() if(Boost_DIR) diff --git a/docker/Dockerfile b/docker/Dockerfile index 459741e98e..1ef4456874 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -12,14 +12,16 @@ ENV DEBIAN_FRONTEND noninteractive WORKDIR /tmp SHELL [ "/bin/bash", "-c" ] +ARG EXTRA_PACKAGES="" + RUN apt-get update && \ apt-get dist-upgrade -y && \ - apt-get install -y build-essential cmake libnuma-dev wget gnupg2 m4 && \ + apt-get install -y build-essential cmake libnuma-dev wget gnupg2 m4 bash-completion git-core && \ wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - && \ echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/debian/ ubuntu main' | tee /etc/apt/sources.list.d/rocm.list && \ apt-get update && \ apt-get dist-upgrade -y && \ - apt-get install -y rocm-dev rocm-utils roctracer-dev rocprofiler-dev hip-base hsa-amd-aqlprofile hsa-rocr-dev hsakmt-roct-dev + apt-get install -y rocm-dev rocm-utils roctracer-dev rocprofiler-dev hip-base hsa-amd-aqlprofile hsa-rocr-dev hsakmt-roct-dev ${EXTRA_PACKAGES} ENV LC_ALL C.UTF-8 WORKDIR /home diff --git a/examples/transpose/transpose.cpp b/examples/transpose/transpose.cpp index 20599937a2..3ca1f16373 100644 --- a/examples/transpose/transpose.cpp +++ b/examples/transpose/transpose.cpp @@ -84,14 +84,14 @@ transpose_a(int* in, int* out, int M, int N) } void -run(int argc, char** argv) +run(int rank, int argc, char** argv) { (void) argc; (void) argv; unsigned int M = 4960; unsigned int N = 4960; - std::cout << "M: " << M << " N: " << N << std::endl; + std::cout << "[" << rank << "] M: " << M << " N: " << N << std::endl; size_t size = sizeof(int) * M * N; int* matrix = (int*) malloc(size); for(size_t i = 0; i < M * N; i++) @@ -130,7 +130,7 @@ run(int argc, char** argv) double time = std::chrono::duration_cast>(t2 - t1).count(); float GB = (float) size * times * 2 / (1 << 30); - std::cout << "Runtime of transpose is " << time << " sec\n" + std::cout << "[" << rank << "] Runtime of transpose is " << time << " sec\n" << "The average performance of transpose is " << GB / time << " GBytes/sec" << std::endl; @@ -151,6 +151,21 @@ run(int argc, char** argv) #if defined(USE_MPI) # include + +void +do_a2a(int rank) +{ + // Define my value + int values[3]; + for(int i = 0; i < 3; ++i) + values[i] = rank * 300 + i * 100; + printf("Process %d, values = %d, %d, %d.\n", rank, values[0], values[1], values[2]); + + int buffer_recv[3]; + MPI_Alltoall(&values, 1, MPI_INT, buffer_recv, 1, MPI_INT, MPI_COMM_WORLD); + printf("Values collected on process %d: %d, %d, %d.\n", rank, buffer_recv[0], + buffer_recv[1], buffer_recv[2]); +} #endif int @@ -161,9 +176,10 @@ main(int argc, char** argv) MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); #endif - if(rank == 0) run(argc, argv); + if(rank == 0) run(rank, argc, argv); #if defined(USE_MPI) MPI_Barrier(MPI_COMM_WORLD); + do_a2a(rank); MPI_Finalize(); #endif return 0; diff --git a/external/timemory b/external/timemory index 9f27fafd2e..11183bbdd7 160000 --- a/external/timemory +++ b/external/timemory @@ -1 +1 @@ -Subproject commit 9f27fafd2e28f8db2b1cc908b2c910a45136b31d +Subproject commit 11183bbdd77c58a3165b08d656ff7336ef6b825b diff --git a/include/hosttrace.hpp b/include/hosttrace.hpp index 98d223417b..3a53e24aec 100644 --- a/include/hosttrace.hpp +++ b/include/hosttrace.hpp @@ -395,7 +395,13 @@ struct module_function module = modname; function = fname; signature = get_func_file_line_info(mod, proc); - assert(proc->isInstrumentable() == true); + if(!proc->isInstrumentable()) + { + verbprintf(0, + "Warning! module function generated for un-instrumentable " + "function: %s [%s]\n", + function.c_str(), module.c_str()); + } std::pair _range{}; if(proc->getAddressRange(_range.first, _range.second)) address_range = _range.second - _range.first; diff --git a/include/library.hpp b/include/library.hpp index 0295b2e884..2a727084ea 100644 --- a/include/library.hpp +++ b/include/library.hpp @@ -115,6 +115,9 @@ struct mpi_gotcha : comp::base // this will get called right after MPI_Init and MPI_Init_thread with the return value void audit(const gotcha_data_t& _data, audit::outgoing, int _retval); + + // this will get called right before MPI_Finalize + void audit(const gotcha_data_t& _data, audit::incoming); }; // timemory api struct diff --git a/scripts/build-release.sh b/scripts/build-release.sh new file mode 100755 index 0000000000..624f758fa1 --- /dev/null +++ b/scripts/build-release.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +VERSION=0.0.3 +ROCM_VERSION=4.3.0 + +STANDARD_ARGS="-DCPACK_GENERATOR=STGZ -DCMAKE_BUILD_TYPE=Release -DHOSTTRACE_BUILD_DYNINST=ON -DTIMEMORY_BUILD_PORTABLE=ON" + +cmake -B build-release/core ${STANDARD_ARGS} -DDYNINST_BUILD_{TBB,BOOST,ELFUTILS,LIBIBERTY}=ON -DDYNINST_USE_OpenMP=OFF -DHOSTTRACE_USE_MPI_HEADERS=ON -DHOSTTRACE_USE_ROCTRACER=OFF . +cmake --build build-release/core --target package --parallel 8 +cp build-release/core/hosttrace-${VERSION}-Linux.sh build-release/hosttrace-${VERSION}-Linux.sh + +cmake -B build-release/rocm-mpi ${STANDARD_ARGS} -DDYNINST_BUILD_{TBB,BOOST,ELFUTILS,LIBIBERTY}=ON -DDYNINST_USE_OpenMP=ON -DHOSTTRACE_USE_MPI_HEADERS=ON -DHOSTTRACE_USE_ROCTRACER=ON . +cmake --build build-release/rocm-mpi --target package --parallel 8 +cp build-release/rocm-mpi/hosttrace-${VERSION}-Linux.sh build-release/hosttrace-${VERSION}-Linux-ROCm-${ROCM_VERSION}.sh diff --git a/scripts/setup-env.sh.in b/scripts/setup-env.sh.in new file mode 100644 index 0000000000..a6925fb489 --- /dev/null +++ b/scripts/setup-env.sh.in @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +BASEDIR=$(dirname ${BASH_SOURCE[0]}) +BASEDIR=$(cd ${BASEDIR}/../.. && pwd) + +if [ ! -d "${BASEDIR}" ]; then + echo "${BASEDIR} does not exist" + return 1 +fi + +export PATH=${BASEDIR}/bin:${PATH} +export LD_LIBRARY_PATH=${BASEDIR}/@CMAKE_INSTALL_LIBDIR@:${LD_LIBRARY_PATH} + +return 0 diff --git a/src/hosttrace.cpp b/src/hosttrace.cpp index f0b89b7ec9..7300ea98a9 100644 --- a/src/hosttrace.cpp +++ b/src/hosttrace.cpp @@ -730,12 +730,14 @@ main(int argc, char** argv) std::set module_names; auto _add_overlapping = [](module_t* mitr, procedure_t* pitr) { + if(!pitr->isInstrumentable()) return; std::vector _overlapping{}; if(pitr->findOverlapping(_overlapping)) { overlapping_module_functions.insert(module_function{ mitr, pitr }); for(auto oitr : _overlapping) { + if(!oitr->isInstrumentable()) continue; overlapping_module_functions.insert( module_function{ oitr->getModule(), oitr }); } @@ -752,7 +754,8 @@ main(int argc, char** argv) { for(auto* pitr : *procedures) { - auto _modfn = module_function(itr, pitr); + if(!pitr->isInstrumentable()) continue; + auto _modfn = module_function{ itr, pitr }; module_names.insert(_modfn.module); available_module_functions.insert(std::move(_modfn)); _add_overlapping(itr, pitr); @@ -771,9 +774,9 @@ main(int argc, char** argv) for(auto* itr : *app_functions) { module_t* mod = itr->getModule(); - if(mod) + if(mod && itr->isInstrumentable()) { - auto _modfn = module_function(mod, itr); + auto _modfn = module_function{ mod, itr }; module_names.insert(_modfn.module); available_module_functions.insert(std::move(_modfn)); _add_overlapping(mod, itr); @@ -1332,11 +1335,11 @@ main(int argc, char** argv) else itr->getModuleName(modname, FUNCNAMELEN); - if(itr == main_func) + if(itr == main_func && main_func->isInstrumentable()) { hash_ids.emplace_back(std::hash()(main_sign.get()), main_sign.get()); - auto main_mf = module_function(modname, fname, main_sign, itr); + auto main_mf = module_function{ modname, fname, main_sign, itr }; available_module_functions.insert(main_mf); instrumented_module_functions.insert(main_mf); continue; @@ -1460,8 +1463,8 @@ main(int argc, char** argv) } hash_ids.emplace_back(std::hash()(name.get()), name.get()); - available_module_functions.insert(module_function(mod, itr)); - instrumented_module_functions.insert(module_function(mod, itr)); + available_module_functions.insert(module_function{ mod, itr }); + instrumented_module_functions.insert(module_function{ mod, itr }); auto _f = [=]() { verbprintf(1, "Instrumenting |> [ %s ] -> [ %s ]\n", modname, diff --git a/src/libmisc.cpp b/src/libmisc.cpp index f82bf44c8d..bc77ca5be8 100644 --- a/src/libmisc.cpp +++ b/src/libmisc.cpp @@ -91,6 +91,17 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::outgoing, int _retval) } } +void +mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming) +{ + HOSTTRACE_DEBUG("[%s] %s()\n", __FUNCTION__, _data.tool_id.c_str()); + if(mpip_index != std::numeric_limits::max()) + comp::deactivate_mpip, hosttrace>( + mpip_index); + hosttrace_pop_trace("MPI_Finalize()"); + hosttrace_trace_finalize(); +} + void hosttrace_component::start() { diff --git a/src/library.cpp b/src/library.cpp index 5e9425488d..c4ae8f4354 100644 --- a/src/library.cpp +++ b/src/library.cpp @@ -83,6 +83,9 @@ setup_gotchas() mpi_gotcha_t::template configure<0, int, int*, char***>("MPI_Init"); mpi_gotcha_t::template configure<1, int, int*, char***, int, int*>( "MPI_Init_thread"); +#if defined(HOSTTRACE_USE_MPI_HEADERS) + mpi_gotcha_t::template configure<3, int>("MPI_Finalize"); +#endif }; } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e8c7b0f30c..9a9effde7a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,18 +1,25 @@ -if(NOT DYNINST_API_RT_DIR AND DYNINST_API_RT) - get_filename_component(DYNINST_API_RT_DIR "${DYNINST_API_RT}" DIRECTORY) +if(NOT HOSTTRACE_DYNINST_API_RT_DIR AND HOSTTRACE_DYNINST_API_RT) + get_filename_component(HOSTTRACE_DYNINST_API_RT_DIR "${HOSTTRACE_DYNINST_API_RT}" + DIRECTORY) endif() if(HOSTTRACE_BUILD_DYNINST) - set(DYNINST_API_RT_DIR + set(HOSTTRACE_DYNINST_API_RT_DIR "${PROJECT_BINARY_DIR}/external/dyninst/dyninstAPI_RT:${PROJECT_BINARY_DIR}/external/dyninst/dyninstAPI" ) endif() set(_test_environment "HOSTTRACE_USE_PERFETTO=ON" - "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}:${DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}") + "HOSTTRACE_USE_TIMEMORY=ON" + "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}:${HOSTTRACE_DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}" + ) if(TARGET transpose) + if(TRANSPOSE_USE_MPI) + set(COMMAND_PREFIX ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 2) + endif() + add_test( NAME transpose-binary-rewrite COMMAND @@ -22,7 +29,7 @@ if(TARGET transpose) add_test( NAME transpose-binary-rewrite-run - COMMAND $/transpose.inst + COMMAND ${COMMAND_PREFIX} $/transpose.inst WORKING_DIRECTORY ${PROJECT_BINARY_DIR}) add_test(