diff --git a/projects/rocprofiler-systems/.cmake-format.yaml b/projects/rocprofiler-systems/.cmake-format.yaml index 8b5d79d794..58db1d2797 100644 --- a/projects/rocprofiler-systems/.cmake-format.yaml +++ b/projects/rocprofiler-systems/.cmake-format.yaml @@ -29,6 +29,12 @@ parse: RUN_ARGS: '*' ENVIRONMENT: '*' LABELS: '*' + PROPERTIES: '*' + omnitrace_target_compile_definitions: + kwargs: + PUBLIC: '*' + PRIVATE: '*' + INTERFACE: '*' override_spec: {} vartags: [] proptags: [] diff --git a/projects/rocprofiler-systems/.github/workflows/linux-ci.yml b/projects/rocprofiler-systems/.github/workflows/linux-ci.yml index 6e56c1a2c2..172844894b 100644 --- a/projects/rocprofiler-systems/.github/workflows/linux-ci.yml +++ b/projects/rocprofiler-systems/.github/workflows/linux-ci.yml @@ -9,6 +9,8 @@ on: env: BUILD_TYPE: Release ELFUTILS_DOWNLOAD_VERSION: 0.183 + OMNITRACE_DEBUG_FINALIZE: ON + OMNITRACE_VERBOSE: 1 jobs: ubuntu-focal: @@ -22,6 +24,7 @@ jobs: - uses: actions/checkout@v2 - name: Install Packages + timeout-minutes: 5 run: sudo apt-get update && sudo apt-get install -y build-essential m4 autoconf libtool python3-pip libtbb-dev libboost-{atomic,system,thread,date-time,filesystem,timer}-dev ${{ matrix.compiler }} ${{ matrix.mpi }} && @@ -36,6 +39,7 @@ jobs: echo "LD_LIBRARY_PATH=/opt/omnitrace/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV - name: Configure CMake + timeout-minutes: 5 run: cmake --version && if [ -z "${{ matrix.mpi }}" ]; then USE_MPI=OFF; else USE_MPI=ON; fi && @@ -102,6 +106,7 @@ jobs: - uses: actions/checkout@v2 - name: Install Packages + timeout-minutes: 5 run: sudo apt-get update && sudo apt-get install -y build-essential m4 autoconf libtool python3-pip ${{ matrix.compiler }} ${{ matrix.mpi }} && @@ -116,6 +121,7 @@ jobs: echo "LD_LIBRARY_PATH=/opt/omnitrace/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV - name: Configure CMake + timeout-minutes: 5 run: cmake --version && if [ -z "${{ matrix.mpi }}" ]; then USE_MPI=OFF; else USE_MPI=ON; fi && @@ -181,6 +187,7 @@ jobs: - uses: actions/checkout@v2 - name: Install Packages + timeout-minutes: 5 run: sudo apt-get update && sudo apt-get install -y build-essential m4 autoconf libtool python3-pip libboost-{atomic,system,thread,date-time,filesystem,timer}-dev libtbb-dev libiberty-dev ${{ matrix.compiler }} && @@ -196,6 +203,7 @@ jobs: echo "LD_LIBRARY_PATH=/opt/omnitrace/lib:/opt/dyninst/lib:/opt/elfutils/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV - name: Install ElfUtils + timeout-minutes: 5 run: pushd external && wget https://sourceware.org/elfutils/ftp/${ELFUTILS_DOWNLOAD_VERSION}/elfutils-${ELFUTILS_DOWNLOAD_VERSION}.tar.bz2 && @@ -208,6 +216,7 @@ jobs: rm -rf elfutils* - name: Install Dyninst + timeout-minutes: 20 run: cmake --version && git submodule update --init external/dyninst && @@ -222,6 +231,7 @@ jobs: rm -rf build - name: Configure CMake + timeout-minutes: 5 run: cmake --version && cmake -B ${{ github.workspace }}/build @@ -284,6 +294,7 @@ jobs: - uses: actions/checkout@v2 - name: Install Packages + timeout-minutes: 5 run: sudo apt-get update && sudo apt-get install -y build-essential m4 autoconf libtool python3-pip ${{ matrix.compiler }} && @@ -299,6 +310,7 @@ jobs: echo "LD_LIBRARY_PATH=/opt/omnitrace/lib:/opt/dyninst/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV - name: Install Dyninst + timeout-minutes: 20 run: cmake --version && git submodule update --init external/dyninst && @@ -318,6 +330,7 @@ jobs: rm -rf build - name: Configure CMake + timeout-minutes: 5 run: cmake --version && cmake -B ${{ github.workspace }}/build @@ -382,6 +395,7 @@ jobs: - uses: actions/checkout@v2 - name: Install Packages + timeout-minutes: 5 run: echo '1' | sudo tee /proc/sys/kernel/perf_event_paranoid && sudo apt-get update && @@ -389,7 +403,7 @@ jobs: sudo wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | sudo apt-key add - && echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${{ matrix.rocm_version }}/ ubuntu main" | sudo tee /etc/apt/sources.list.d/rocm.list && sudo apt-get update && - sudo apt-get install -y build-essential m4 autoconf libtool python3-pip libboost-{atomic,system,thread,date-time,filesystem,timer}-dev libtbb-dev libiberty-dev ${{ matrix.compiler }} libnuma-dev rocm-dev rocm-utils roctracer-dev rocprofiler-dev hip-base hsa-amd-aqlprofile hsa-rocr-dev hsakmt-roct-dev ${{ matrix.mpi }} libpapi-dev && + sudo apt-get install -y build-essential m4 autoconf libtool python3-pip libboost-{atomic,system,thread,date-time,filesystem,timer}-dev libtbb-dev libiberty-dev ${{ matrix.compiler }} libudev-dev libnuma-dev rocm-dev rocm-utils roctracer-dev rocprofiler-dev hip-base hsa-amd-aqlprofile hsa-rocr-dev hsakmt-roct-dev ${{ matrix.mpi }} libpapi-dev && sudo python3 -m pip install --upgrade pip && python3 -m pip install 'cmake==3.16.3' @@ -402,6 +416,7 @@ jobs: echo "LD_LIBRARY_PATH=/opt/omnitrace/lib:/opt/dyninst/lib:/opt/elfutils/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV - name: Install ElfUtils + timeout-minutes: 5 run: pushd external && wget https://sourceware.org/elfutils/ftp/${ELFUTILS_DOWNLOAD_VERSION}/elfutils-${ELFUTILS_DOWNLOAD_VERSION}.tar.bz2 && @@ -414,6 +429,7 @@ jobs: rm -rf elfutils* - name: Install Dyninst + timeout-minutes: 20 run: cmake --version && git submodule update --init external/dyninst && @@ -428,6 +444,7 @@ jobs: rm -rf build - name: Configure CMake + timeout-minutes: 5 run: cmake --version && cmake -B ${{ github.workspace }}/build diff --git a/projects/rocprofiler-systems/CMakeLists.txt b/projects/rocprofiler-systems/CMakeLists.txt index 0020562b41..6d113a494b 100644 --- a/projects/rocprofiler-systems/CMakeLists.txt +++ b/projects/rocprofiler-systems/CMakeLists.txt @@ -69,6 +69,9 @@ omnitrace_add_option(OMNITRACE_USE_MPI "Enable MPI support" OFF) omnitrace_add_option(OMNITRACE_USE_HIP "Enable HIP support" ON) omnitrace_add_option(OMNITRACE_USE_ROCTRACER "Enable roctracer support" ${OMNITRACE_USE_HIP}) +omnitrace_add_option( + OMNITRACE_USE_ROCM_SMI "Enable rocm-smi support for power/temp/etc. sampling" + ${OMNITRACE_USE_HIP}) omnitrace_add_option(OMNITRACE_USE_MPI_HEADERS "Enable wrapping MPI functions w/o enabling MPI dependency" OFF) omnitrace_add_option(OMNITRACE_BUILD_DYNINST "Build dyninst from submodule" OFF) @@ -139,7 +142,12 @@ if(OMNITRACE_BUILD_HIDDEN_VISIBILITY) set(CMAKE_VISIBILITY_INLINES_HIDDEN ON) endif() +if(NOT DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION) + set(CMAKE_INTERPROCEDURAL_OPTIMIZATION OFF) +endif() + if(OMNITRACE_BUILD_LTO) + omnitrace_save_variables(LTO VARIABLES CMAKE_INTERPROCEDURAL_OPTIMIZATION) set(CMAKE_INTERPROCEDURAL_OPTIMIZATION ON) endif() @@ -153,10 +161,12 @@ set(library_sources ${CMAKE_CURRENT_LIST_DIR}/src/library.cpp ${CMAKE_CURRENT_LIST_DIR}/src/library/config.cpp ${CMAKE_CURRENT_LIST_DIR}/src/library/critical_trace.cpp + ${CMAKE_CURRENT_LIST_DIR}/src/library/kokkosp.cpp ${CMAKE_CURRENT_LIST_DIR}/src/library/gpu.cpp ${CMAKE_CURRENT_LIST_DIR}/src/library/perfetto.cpp ${CMAKE_CURRENT_LIST_DIR}/src/library/ptl.cpp ${CMAKE_CURRENT_LIST_DIR}/src/library/sampling.cpp + ${CMAKE_CURRENT_LIST_DIR}/src/library/state.cpp ${CMAKE_CURRENT_LIST_DIR}/src/library/thread_data.cpp ${CMAKE_CURRENT_LIST_DIR}/src/library/timemory.cpp ${CMAKE_CURRENT_LIST_DIR}/src/library/components/backtrace.cpp @@ -185,6 +195,9 @@ set(library_headers ${CMAKE_CURRENT_LIST_DIR}/include/library/components/fork_gotcha.hpp ${CMAKE_CURRENT_LIST_DIR}/include/library/components/mpi_gotcha.hpp ${CMAKE_CURRENT_LIST_DIR}/include/library/components/omnitrace.hpp + ${CMAKE_CURRENT_LIST_DIR}/include/library/components/rocm_smi.hpp + ${CMAKE_CURRENT_LIST_DIR}/include/library/components/roctracer.hpp + ${CMAKE_CURRENT_LIST_DIR}/include/library/components/roctracer_callbacks.hpp ${CMAKE_CURRENT_LIST_DIR}/include/library/components/pthread_gotcha.hpp ${perfetto_DIR}/sdk/perfetto.h) @@ -193,11 +206,13 @@ add_library(omnitrace-library SHARED ${library_sources} ${library_headers}) if(OMNITRACE_USE_ROCTRACER) target_sources( omnitrace-library - PRIVATE - ${CMAKE_CURRENT_LIST_DIR}/src/library/components/roctracer.cpp - ${CMAKE_CURRENT_LIST_DIR}/src/library/components/roctracer_callbacks.cpp - ${CMAKE_CURRENT_LIST_DIR}/include/library/components/roctracer.hpp - ${CMAKE_CURRENT_LIST_DIR}/include/library/components/roctracer_callbacks.hpp) + PRIVATE ${CMAKE_CURRENT_LIST_DIR}/src/library/components/roctracer.cpp + ${CMAKE_CURRENT_LIST_DIR}/src/library/components/roctracer_callbacks.cpp) +endif() + +if(OMNITRACE_USE_ROCM_SMI) + target_sources(omnitrace-library + PRIVATE ${CMAKE_CURRENT_LIST_DIR}/src/library/components/rocm_smi.cpp) endif() target_include_directories(omnitrace-library SYSTEM PRIVATE ${perfetto_DIR}/sdk) @@ -214,6 +229,7 @@ target_link_libraries( $ $ $ + $ $ $ $ @@ -285,6 +301,11 @@ set_target_properties( "\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}:\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}/timemory/libunwind:\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}/dyninst-tpls/lib" ) +if(CMAKE_BUILD_TYPE MATCHES "^(DEBUG|Debug)") + string(REPLACE " " ";" _FLAGS "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}") + target_compile_options(omnitrace-exe PRIVATE ${_FLAGS}) +endif() + install( TARGETS omnitrace-exe DESTINATION ${CMAKE_INSTALL_BINDIR} @@ -320,6 +341,10 @@ install( # # ------------------------------------------------------------------------------# +if(OMNITRACE_BUILD_LTO) + omnitrace_restore_variables(LTO VARIABLES CMAKE_INTERPROCEDURAL_OPTIMIZATION) +endif() + if(OMNITRACE_BUILD_TESTING) add_subdirectory(examples) endif() diff --git a/projects/rocprofiler-systems/cmake/Compilers.cmake b/projects/rocprofiler-systems/cmake/Compilers.cmake index 54262d87b3..289d0b2d8a 100644 --- a/projects/rocprofiler-systems/cmake/Compilers.cmake +++ b/projects/rocprofiler-systems/cmake/Compilers.cmake @@ -457,6 +457,9 @@ endfunction() # ----------------------------------------------------------------------------------------# function(OMNITRACE_TARGET_COMPILE_DEFINITIONS _TARG _VIS) foreach(_DEF ${ARGN}) + if(NOT "${_DEF}" MATCHES "[A-Za-z_]+=.*" AND "${_DEF}" MATCHES "^OMNITRACE_") + set(_DEF "${_DEF}=1") + endif() target_compile_definitions(${_TARG} ${_VIS} $<$:${_DEF}>) if(CMAKE_CUDA_COMPILER_IS_NVIDIA) target_compile_definitions(${_TARG} ${_VIS} diff --git a/projects/rocprofiler-systems/cmake/Formatting.cmake b/projects/rocprofiler-systems/cmake/Formatting.cmake index 44378e7537..444764d109 100644 --- a/projects/rocprofiler-systems/cmake/Formatting.cmake +++ b/projects/rocprofiler-systems/cmake/Formatting.cmake @@ -42,7 +42,8 @@ find_program(OMNITRACE_CLANG_FORMAT_EXE NAMES clang-format-11 clang-format-mp-11 if(OMNITRACE_CLANG_FORMAT_EXE) file(GLOB_RECURSE sources ${PROJECT_SOURCE_DIR}/src/*.cpp) - file(GLOB_RECURSE headers ${PROJECT_SOURCE_DIR}/include/*.hpp) + file(GLOB_RECURSE headers ${PROJECT_SOURCE_DIR}/include/*.hpp + ${PROJECT_SOURCE_DIR}/include/*.hpp.in) file(GLOB_RECURSE examples ${PROJECT_SOURCE_DIR}/examples/*.cpp ${PROJECT_SOURCE_DIR}/examples/*.hpp) file(GLOB_RECURSE external ${PROJECT_SOURCE_DIR}/examples/lulesh/external/*.cpp diff --git a/projects/rocprofiler-systems/cmake/Modules/FindMPI-Headers.cmake b/projects/rocprofiler-systems/cmake/Modules/FindMPI-Headers.cmake index 87823dad00..5753e29918 100644 --- a/projects/rocprofiler-systems/cmake/Modules/FindMPI-Headers.cmake +++ b/projects/rocprofiler-systems/cmake/Modules/FindMPI-Headers.cmake @@ -9,7 +9,7 @@ include(FindPackageHandleStandardArgs) find_path( MPI_HEADERS_INCLUDE_DIR NAMES mpi.h - PATH_SUFFIXES include include/mpich include/openmpi mpich openmpi + PATH_SUFFIXES include/openmpi openmpi include include/mpich mpich HINTS ${MPI_ROOT_DIR} PATHS ${MPI_ROOT_DIR}) diff --git a/projects/rocprofiler-systems/cmake/Modules/Findrocm-smi.cmake b/projects/rocprofiler-systems/cmake/Modules/Findrocm-smi.cmake new file mode 100644 index 0000000000..034891d970 --- /dev/null +++ b/projects/rocprofiler-systems/cmake/Modules/Findrocm-smi.cmake @@ -0,0 +1,70 @@ +# Distributed under the OSI-approved BSD 3-Clause License. See accompanying file +# Copyright.txt or https://cmake.org/licensing for details. + +include(FindPackageHandleStandardArgs) + +# ----------------------------------------------------------------------------------------# + +# set(_ROCM_PATHS $ENV{ROCM_HOME} /opt/rocm) + +# ----------------------------------------------------------------------------------------# + +find_path( + rocm-smi_ROOT_DIR + NAMES include/rocm_smi/rocm_smi.h + HINTS ${_ROCM_PATHS} + PATHS ${_ROCM_PATHS} + PATH_SUFFIXES rocm_smi) + +mark_as_advanced(rocm-smi_ROOT_DIR) + +# ----------------------------------------------------------------------------------------# + +find_path( + rocm-smi_INCLUDE_DIR + NAMES rocm_smi/rocm_smi.h + HINTS ${rocm-smi_ROOT_DIR} ${_ROCM_PATHS} + PATHS ${rocm-smi_ROOT_DIR} ${_ROCM_PATHS} + PATH_SUFFIXES rocm_smi/include rocm_smi) + +mark_as_advanced(rocm-smi_INCLUDE_DIR) + +# ----------------------------------------------------------------------------------------# + +find_library( + rocm-smi_LIBRARY + NAMES rocm_smi64 rocm_smi + HINTS ${rocm-smi_ROOT_DIR} ${_ROCM_PATHS} + PATHS ${rocm-smi_ROOT_DIR} ${_ROCM_PATHS} + PATH_SUFFIXES rocm_smi/lib rocm_smi/lib64 lib lib64) + +if(rocm-smi_LIBRARY) + get_filename_component(rocm-smi_LIBRARY_DIR "${rocm-smi_LIBRARY}" PATH CACHE) +endif() + +mark_as_advanced(rocm-smi_LIBRARY) + +# ----------------------------------------------------------------------------------------# + +find_package_handle_standard_args(rocm-smi DEFAULT_MSG rocm-smi_ROOT_DIR + rocm-smi_INCLUDE_DIR rocm-smi_LIBRARY) + +# ------------------------------------------------------------------------------# + +if(rocm-smi_FOUND) + add_library(rocm-smi::rocm-smi INTERFACE IMPORTED) + add_library(rocm-smi::roctx INTERFACE IMPORTED) + set(rocm-smi_INCLUDE_DIRS ${rocm-smi_INCLUDE_DIR}) + set(rocm-smi_LIBRARIES ${rocm-smi_LIBRARY}) + set(rocm-smi_LIBRARY_DIRS ${rocm-smi_LIBRARY_DIR}) + + target_include_directories(rocm-smi::rocm-smi INTERFACE ${rocm-smi_INCLUDE_DIR}) + target_link_libraries(rocm-smi::rocm-smi INTERFACE ${rocm-smi_LIBRARY}) + +endif() + +# ------------------------------------------------------------------------------# + +unset(_ROCM_PATHS) + +# ------------------------------------------------------------------------------# diff --git a/projects/rocprofiler-systems/cmake/Modules/Findroctracer.cmake b/projects/rocprofiler-systems/cmake/Modules/Findroctracer.cmake index e085fb62fc..2ef0281909 100644 --- a/projects/rocprofiler-systems/cmake/Modules/Findroctracer.cmake +++ b/projects/rocprofiler-systems/cmake/Modules/Findroctracer.cmake @@ -61,18 +61,25 @@ find_library( PATHS ${roctracer_ROOT_DIR} ${_ROCM_PATHS} PATH_SUFFIXES lib lib64) -find_library( - roctracer_hsakmt_LIBRARY - NAMES hsakmt - HINTS ${roctracer_ROOT_DIR} ${_ROCM_PATHS} - PATHS ${roctracer_ROOT_DIR} ${_ROCM_PATHS} - PATH_SUFFIXES lib lib64) +find_package(hsakmt) +if(hsakmt_FOUND) + set(roctracer_hsakmt_LIBRARY + hsakmt::hsakmt + CACHE STRING "Imported hsakmt target") +else() + find_library( + roctracer_hsakmt_LIBRARY + NAMES hsakmt + HINTS ${roctracer_ROOT_DIR} ${_ROCM_PATHS} + PATHS ${roctracer_ROOT_DIR} ${_ROCM_PATHS} + PATH_SUFFIXES lib lib64) +endif() if(roctracer_LIBRARY) get_filename_component(roctracer_LIBRARY_DIR "${roctracer_LIBRARY}" PATH CACHE) endif() -mark_as_advanced(roctracer_LIBRARY roctracer_roctx_LIBRARY) +mark_as_advanced(roctracer_LIBRARY roctracer_roctx_LIBRARY roctracer_hsakmt_LIBRARY) # ----------------------------------------------------------------------------------------# diff --git a/projects/rocprofiler-systems/cmake/Packages.cmake b/projects/rocprofiler-systems/cmake/Packages.cmake index 7d89e9a01c..7c2f7af55a 100644 --- a/projects/rocprofiler-systems/cmake/Packages.cmake +++ b/projects/rocprofiler-systems/cmake/Packages.cmake @@ -16,6 +16,8 @@ omnitrace_add_interface_library( omnitrace_add_interface_library(omnitrace-hip "Provides flags and libraries for HIP") omnitrace_add_interface_library(omnitrace-roctracer "Provides flags and libraries for roctracer") +omnitrace_add_interface_library(omnitrace-rocm-smi + "Provides flags and libraries for rocm-smi") omnitrace_add_interface_library(omnitrace-mpi "Provides MPI or MPI headers") omnitrace_add_interface_library(omnitrace-ptl "Enables PTL support (tasking)") @@ -60,7 +62,7 @@ endif() if(OMNITRACE_USE_HIP) list(APPEND CMAKE_PREFIX_PATH /opt/rocm) find_package(hip ${omnitrace_FIND_QUIETLY} REQUIRED) - target_compile_definitions(omnitrace-hip INTERFACE OMNITRACE_USE_HIP) + omnitrace_target_compile_definitions(omnitrace-hip INTERFACE OMNITRACE_USE_HIP) target_link_libraries(omnitrace-hip INTERFACE hip::host) endif() @@ -73,12 +75,28 @@ endif() if(OMNITRACE_USE_ROCTRACER) list(APPEND CMAKE_PREFIX_PATH /opt/rocm) find_package(roctracer ${omnitrace_FIND_QUIETLY} REQUIRED) - target_compile_definitions(omnitrace-roctracer INTERFACE OMNITRACE_USE_ROCTRACER) + omnitrace_target_compile_definitions(omnitrace-roctracer + INTERFACE OMNITRACE_USE_ROCTRACER) target_link_libraries(omnitrace-roctracer INTERFACE roctracer::roctracer omnitrace::omnitrace-hip) set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${roctracer_LIBRARY_DIRS}") endif() +# ----------------------------------------------------------------------------------------# +# +# rocm-smmi +# +# ----------------------------------------------------------------------------------------# + +if(OMNITRACE_USE_ROCM_SMI) + list(APPEND CMAKE_PREFIX_PATH /opt/rocm) + find_package(rocm-smi ${omnitrace_FIND_QUIETLY} REQUIRED) + omnitrace_target_compile_definitions(omnitrace-rocm-smi + INTERFACE OMNITRACE_USE_ROCM_SMI) + target_link_libraries(omnitrace-rocm-smi INTERFACE rocm-smi::rocm-smi) + set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${rocm-smi_LIBRARY_DIRS}") +endif() + # ----------------------------------------------------------------------------------------# # # MPI @@ -90,8 +108,8 @@ if(OMNITRACE_USE_MPI) target_link_libraries(omnitrace-mpi INTERFACE MPI::MPI_C MPI::MPI_CXX) elseif(OMNITRACE_USE_MPI_HEADERS) find_package(MPI-Headers ${omnitrace_FIND_QUIETLY} REQUIRED) - target_compile_definitions(omnitrace-mpi INTERFACE TIMEMORY_USE_MPI_HEADERS - OMNITRACE_USE_MPI_HEADERS) + omnitrace_target_compile_definitions( + omnitrace-mpi INTERFACE TIMEMORY_USE_MPI_HEADERS OMNITRACE_USE_MPI_HEADERS) target_link_libraries(omnitrace-mpi INTERFACE MPI::MPI_HEADERS) endif() @@ -143,7 +161,7 @@ if(OMNITRACE_BUILD_DYNINST) ) if(OMNITRACE_DYNINST_API_RT) - target_compile_definitions( + omnitrace_target_compile_definitions( omnitrace-dyninst INTERFACE DYNINST_API_RT="${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}:$:${CMAKE_INSTALL_PREFIX}/lib/$:$" @@ -163,7 +181,7 @@ else() PATH_SUFFIXES lib) if(OMNITRACE_DYNINST_API_RT) - target_compile_definitions( + omnitrace_target_compile_definitions( omnitrace-dyninst INTERFACE DYNINST_API_RT="${OMNITRACE_DYNINST_API_RT}") endif() @@ -214,7 +232,7 @@ else() endif() if(OMNITRACE_DYNINST_API_RT) - target_compile_definitions( + omnitrace_target_compile_definitions( omnitrace-dyninst INTERFACE DYNINST_API_RT="${OMNITRACE_DYNINST_API_RT}") endif() @@ -253,7 +271,8 @@ else() target_include_directories( omnitrace-dyninst SYSTEM INTERFACE ${TBB_INCLUDE_DIR} ${Boost_INCLUDE_DIRS} ${DYNINST_HEADER_DIR}) - target_compile_definitions(omnitrace-dyninst INTERFACE omnitrace_USE_DYNINST) + omnitrace_target_compile_definitions(omnitrace-dyninst + INTERFACE OMNITRACE_USE_DYNINST) endif() endif() diff --git a/projects/rocprofiler-systems/examples/lulesh/CMakeLists.txt b/projects/rocprofiler-systems/examples/lulesh/CMakeLists.txt index f90d576d46..ffe64737ae 100644 --- a/projects/rocprofiler-systems/examples/lulesh/CMakeLists.txt +++ b/projects/rocprofiler-systems/examples/lulesh/CMakeLists.txt @@ -4,7 +4,16 @@ project(lulesh LANGUAGES C CXX) list(INSERT CMAKE_MODULE_PATH 0 ${PROJECT_SOURCE_DIR}/cmake/Modules) -add_subdirectory(external) +option(LULESH_BUILD_KOKKOS "Build Kokkos from submodule" ON) +if(LULESH_BUILD_KOKKOS) + add_subdirectory(external) + if(LULESH_USE_CUDA OR LULESH_USE_HIP) + kokkos_compilation(PROJECT) + endif() +else() + find_package(Kokkos REQUIRED COMPONENTS separable_compilation) + kokkos_compilation(PROJECT) +endif() set(CMAKE_CXX_EXTENSIONS OFF) @@ -14,7 +23,9 @@ if("${CMAKE_BUILD_TYPE}" STREQUAL "") CACHE STRING "CMake build type" FORCE) endif() -if(DEFINED OMNITRACE_USE_MPI) +if(DEFINED OMNITRACE_USE_MPI + AND NOT LULESH_USE_CUDA + AND NOT LULESH_USE_HIP) option(LULESH_USE_MPI "Enable MPI" ${OMNITRACE_USE_MPI}) else() option(LULESH_USE_MPI "Enable MPI" OFF) diff --git a/projects/rocprofiler-systems/examples/lulesh/external/CMakeLists.txt b/projects/rocprofiler-systems/examples/lulesh/external/CMakeLists.txt index 7ede9ec559..2329f18eb1 100644 --- a/projects/rocprofiler-systems/examples/lulesh/external/CMakeLists.txt +++ b/projects/rocprofiler-systems/examples/lulesh/external/CMakeLists.txt @@ -1,22 +1,27 @@ +# include(Utilities) + +option(LULESH_USE_CUDA "Enable Kokkos CUDA backend for lulesh" OFF) +option(LULESH_USE_HIP "Enable Kokkos HIP backend for lulesh" OFF) + set(Kokkos_ENABLE_SERIAL ON CACHE BOOL "Enable Serial") -set(Kokkos_ENABLE_OPENMP - ON - CACHE BOOL "Enable OpenMP") -if(USE_CUDA) + +if(LULESH_USE_CUDA) set(Kokkos_ENABLE_CUDA ON - CACHE BOOL "Enable CUDA") - set(Kokkos_ENABLE_CUDA_UVM - ON - CACHE BOOL "Enable CUDA UVM") + CACHE BOOL "Enable CUDA" FORCE) set(Kokkos_ENABLE_CUDA_LAMBDA ON - CACHE BOOL "Enable CUDA UVM") - set(Kokkos_ENABLE_CUDA_CONSTEXPR + CACHE BOOL "Enable CUDA lambda support" FORCE) +elseif(LULESH_USE_HIP) + set(Kokkos_ENABLE_HIP ON - CACHE BOOL "Enable CUDA UVM") + CACHE BOOL "Enable HIP" FORCE) +else() + set(Kokkos_ENABLE_OPENMP + ON + CACHE BOOL "Enable OpenMP") endif() checkout_git_submodule( diff --git a/projects/rocprofiler-systems/examples/lulesh/external/kokkos b/projects/rocprofiler-systems/examples/lulesh/external/kokkos index 56468253ef..698a67731a 160000 --- a/projects/rocprofiler-systems/examples/lulesh/external/kokkos +++ b/projects/rocprofiler-systems/examples/lulesh/external/kokkos @@ -1 +1 @@ -Subproject commit 56468253ef601e6968dd1e6714dea8ee26a561c3 +Subproject commit 698a67731a4c3b20ef4fcb728298176e49474033 diff --git a/projects/rocprofiler-systems/examples/lulesh/lulesh-comm.cc b/projects/rocprofiler-systems/examples/lulesh/lulesh-comm.cc index c083bd3900..90a225d164 100644 --- a/projects/rocprofiler-systems/examples/lulesh/lulesh-comm.cc +++ b/projects/rocprofiler-systems/examples/lulesh/lulesh-comm.cc @@ -9,8 +9,8 @@ /* Comm Routines */ # define ALLOW_UNPACKED_PLANE false -# define ALLOW_UNPACKED_ROW false -# define ALLOW_UNPACKED_COL false +# define ALLOW_UNPACKED_ROW false +# define ALLOW_UNPACKED_COL false /* There are coherence issues for packing and unpacking message @@ -59,8 +59,7 @@ void CommRecv(Domain& domain, int msgType, Index_t xferFields, Index_t dx, Index_t dy, Index_t dz, bool doRecv, bool planeOnly) { - if(domain.numRanks() == 1) - return; + if(domain.numRanks() == 1) return; /* post recieve buffers for all incoming messages */ int myRank; @@ -366,8 +365,7 @@ void CommSend(Domain& domain, int msgType, Index_t xferFields, Domain_member* fieldData, Index_t dx, Index_t dy, Index_t dz, bool doSend, bool planeOnly) { - if(domain.numRanks() == 1) - return; + if(domain.numRanks() == 1) return; /* post recieve buffers for all incoming messages */ int myRank; @@ -926,8 +924,7 @@ CommSend(Domain& domain, int msgType, Index_t xferFields, Domain_member* fieldDa void CommSBN(Domain& domain, int xferFields, Domain_member* fieldData) { - if(domain.numRanks() == 1) - return; + if(domain.numRanks() == 1) return; /* summation order should be from smallest value to largest */ /* or we could try out kahan summation! */ @@ -1405,8 +1402,7 @@ CommSBN(Domain& domain, int xferFields, Domain_member* fieldData) void CommSyncPosVel(Domain& domain) { - if(domain.numRanks() == 1) - return; + if(domain.numRanks() == 1) return; int myRank; bool doRecv = false; @@ -1893,8 +1889,7 @@ CommSyncPosVel(Domain& domain) void CommMonoQ(Domain& domain) { - if(domain.numRanks() == 1) - return; + if(domain.numRanks() == 1) return; int myRank; Index_t xferFields = 3; /* delv_xi, delv_eta, delv_zeta */ diff --git a/projects/rocprofiler-systems/examples/lulesh/lulesh-init.cc b/projects/rocprofiler-systems/examples/lulesh/lulesh-init.cc index d28478afee..55d149e8b2 100644 --- a/projects/rocprofiler-systems/examples/lulesh/lulesh-init.cc +++ b/projects/rocprofiler-systems/examples/lulesh/lulesh-init.cc @@ -2,9 +2,6 @@ #if USE_MPI # include #endif -#if _OPENMP -# include -#endif #include "lulesh.h" #include #include @@ -13,12 +10,12 @@ #include static KOKKOS_INLINE_FUNCTION Real_t - CalcElemVolume(const Real_t x0, const Real_t x1, const Real_t x2, const Real_t x3, - const Real_t x4, const Real_t x5, const Real_t x6, const Real_t x7, - const Real_t y0, const Real_t y1, const Real_t y2, const Real_t y3, - const Real_t y4, const Real_t y5, const Real_t y6, const Real_t y7, - const Real_t z0, const Real_t z1, const Real_t z2, const Real_t z3, - const Real_t z4, const Real_t z5, const Real_t z6, const Real_t z7) +CalcElemVolume(const Real_t x0, const Real_t x1, const Real_t x2, const Real_t x3, + const Real_t x4, const Real_t x5, const Real_t x6, const Real_t x7, + const Real_t y0, const Real_t y1, const Real_t y2, const Real_t y3, + const Real_t y4, const Real_t y5, const Real_t y6, const Real_t y7, + const Real_t z0, const Real_t z1, const Real_t z2, const Real_t z3, + const Real_t z4, const Real_t z5, const Real_t z6, const Real_t z7) { Real_t twelveth = Real_t(1.0) / Real_t(12.0); @@ -122,15 +119,15 @@ Domain::Domain(Int_t numRanks, Index_t colLoc, Index_t rowLoc, Index_t planeLoc, , m_dvovmax(Real_t(0.1)) , m_refdens(Real_t(1.0)) , - // - // set pointers to (potentially) "new'd" arrays to null to - // simplify deallocation. - // - m_regNumList(0) -, m_nodeElemStart(0) -, m_nodeElemCornerList(0) -, m_regElemSize(0) -, m_regElemlist(0) +// +// set pointers to (potentially) "new'd" arrays to null to +// simplify deallocation. +// +m_regNumList(0) +// m_nodeElemStart(0), +// m_nodeElemCornerList(0), +// m_regElemSize(0), +// m_regElemlist(0) #if USE_MPI , commDataSend(0) , commDataRecv(0) @@ -171,48 +168,27 @@ Domain::Domain(Int_t numRanks, Index_t colLoc, Index_t rowLoc, Index_t planeLoc, SetupCommBuffers(edgeNodes); // Basic Field Initialization - for(Index_t i = 0; i < numElem(); ++i) - { - e(i) = Real_t(0.0); - p(i) = Real_t(0.0); - q(i) = Real_t(0.0); - ss(i) = Real_t(0.0); - } + Kokkos::deep_copy(m_e, 0.0); + Kokkos::deep_copy(m_p, 0.0); + Kokkos::deep_copy(m_q, 0.0); + Kokkos::deep_copy(m_ss, 0.0); // Note - v initializes to 1.0, not 0.0! - for(Index_t i = 0; i < numElem(); ++i) - { - v(i) = Real_t(1.0); - } + Kokkos::deep_copy(m_v, 1.0); - for(Index_t i = 0; i < numNode(); ++i) - { - xd(i) = Real_t(0.0); - yd(i) = Real_t(0.0); - zd(i) = Real_t(0.0); - } + Kokkos::deep_copy(m_xd, 0.0); + Kokkos::deep_copy(m_yd, 0.0); + Kokkos::deep_copy(m_zd, 0.0); - for(Index_t i = 0; i < numNode(); ++i) - { - xdd(i) = Real_t(0.0); - ydd(i) = Real_t(0.0); - zdd(i) = Real_t(0.0); - } + Kokkos::deep_copy(m_xdd, 0.0); + Kokkos::deep_copy(m_ydd, 0.0); + Kokkos::deep_copy(m_zdd, 0.0); - for(Index_t i = 0; i < numNode(); ++i) - { - nodalMass(i) = Real_t(0.0); - } + Kokkos::deep_copy(m_nodalMass, 0.0); BuildMesh(nx, edgeNodes, edgeElems); -#if _OPENMP SetupThreadSupportStructures(); -#else - // These arrays are not used if we're not threaded - m_nodeElemStart = NULL; - m_nodeElemCornerList = NULL; -#endif // Setup region index sets. For now, these are constant sized // throughout the run, but could be changed every cycle to @@ -247,30 +223,46 @@ Domain::Domain(Int_t numRanks, Index_t colLoc, Index_t rowLoc, Index_t planeLoc, time() = Real_t(0.); cycle() = Int_t(0); + // With C++17 requirement we could just run this on the device + // without creating temporary host copies + auto h_nodelist = Kokkos::create_mirror_view(m_nodelist); + auto h_x = Kokkos::create_mirror_view(m_x); + auto h_y = Kokkos::create_mirror_view(m_y); + auto h_z = Kokkos::create_mirror_view(m_z); + auto h_volo = Kokkos::create_mirror_view(m_volo); + auto h_elemMass = Kokkos::create_mirror_view(m_elemMass); + auto h_nodalMass = Kokkos::create_mirror_view(m_nodalMass); + Kokkos::deep_copy(h_nodelist, m_nodelist); + Kokkos::deep_copy(h_x, m_x); + Kokkos::deep_copy(h_y, m_y); + Kokkos::deep_copy(h_z, m_z); // initialize field data for(Index_t i = 0; i < numElem(); ++i) { - Real_t x_local[8], y_local[8], z_local[8]; - Index_t* elemToNode = nodelist(i); + Real_t x_local[8], y_local[8], z_local[8]; for(Index_t lnode = 0; lnode < 8; ++lnode) { - Index_t gnode = elemToNode[lnode]; - x_local[lnode] = x(gnode); - y_local[lnode] = y(gnode); - z_local[lnode] = z(gnode); + Index_t gnode = h_nodelist(i, lnode); + x_local[lnode] = h_x(gnode); + y_local[lnode] = h_y(gnode); + z_local[lnode] = h_z(gnode); } // volume calculations Real_t volume = CalcElemVolume(x_local, y_local, z_local); - volo(i) = volume; - elemMass(i) = volume; + h_volo(i) = volume; + h_elemMass(i) = volume; for(Index_t j = 0; j < 8; ++j) { - Index_t idx = elemToNode[j]; - nodalMass(idx) += volume / Real_t(8.0); + Index_t idx = h_nodelist(i, j); + h_nodalMass(idx) += volume / Real_t(8.0); } } + Kokkos::deep_copy(m_volo, h_volo); + Kokkos::deep_copy(m_elemMass, h_elemMass); + Kokkos::deep_copy(m_nodalMass, h_nodalMass); + // deposit initial energy // An energy of 3.948746e+7 is correct for a problem with // 45 zones along a side - we need to scale it @@ -281,10 +273,11 @@ Domain::Domain(Int_t numRanks, Index_t colLoc, Index_t rowLoc, Index_t planeLoc, { // Dump into the first zone (which we know is in the corner) // of the domain that sits at the origin - e(0) = einit; + Kokkos::deep_copy(Kokkos::subview(m_e, 0), einit); + // e(0) = einit; } // set initial deltatime base on analytic CFL calculation - deltatime() = (Real_t(.5) * cbrt(volo(0))) / sqrt(Real_t(2.0) * einit); + deltatime() = (Real_t(.5) * cbrt(h_volo(0))) / sqrt(Real_t(2.0) * einit); } // End constructor @@ -315,6 +308,10 @@ Domain::BuildMesh(Int_t nx, Int_t edgeNodes, Int_t edgeElems) { Index_t meshEdgeElems = m_tp * nx; + auto h_x = Kokkos::create_mirror_view(m_x); + auto h_y = Kokkos::create_mirror_view(m_y); + auto h_z = Kokkos::create_mirror_view(m_z); + // initialize nodal coordinates Index_t nidx = 0; Real_t tz = Real_t(1.125) * Real_t(m_planeLoc * nx) / Real_t(meshEdgeElems); @@ -326,9 +323,9 @@ Domain::BuildMesh(Int_t nx, Int_t edgeNodes, Int_t edgeElems) Real_t tx = Real_t(1.125) * Real_t(m_colLoc * nx) / Real_t(meshEdgeElems); for(Index_t col = 0; col < edgeNodes; ++col) { - x(nidx) = tx; - y(nidx) = ty; - z(nidx) = tz; + h_x(nidx) = tx; + h_y(nidx) = ty; + h_z(nidx) = tz; ++nidx; // tx += ds ; // may accumulate roundoff... tx = Real_t(1.125) * Real_t(m_colLoc * nx + col + 1) / @@ -341,6 +338,11 @@ Domain::BuildMesh(Int_t nx, Int_t edgeNodes, Int_t edgeElems) tz = Real_t(1.125) * Real_t(m_planeLoc * nx + plane + 1) / Real_t(meshEdgeElems); } + Kokkos::deep_copy(m_x, h_x); + Kokkos::deep_copy(m_y, h_y); + Kokkos::deep_copy(m_z, h_z); + + auto h_nodelist = Kokkos::create_mirror_view(m_nodelist); // embed hexehedral elements in nodal point lattice Index_t zidx = 0; nidx = 0; @@ -350,15 +352,14 @@ Domain::BuildMesh(Int_t nx, Int_t edgeNodes, Int_t edgeElems) { for(Index_t col = 0; col < edgeElems; ++col) { - Index_t* localNode = nodelist(zidx); - localNode[0] = nidx; - localNode[1] = nidx + 1; - localNode[2] = nidx + edgeNodes + 1; - localNode[3] = nidx + edgeNodes; - localNode[4] = nidx + edgeNodes * edgeNodes; - localNode[5] = nidx + edgeNodes * edgeNodes + 1; - localNode[6] = nidx + edgeNodes * edgeNodes + edgeNodes + 1; - localNode[7] = nidx + edgeNodes * edgeNodes + edgeNodes; + h_nodelist(zidx, 0) = nidx; + h_nodelist(zidx, 1) = nidx + 1; + h_nodelist(zidx, 2) = nidx + edgeNodes + 1; + h_nodelist(zidx, 3) = nidx + edgeNodes; + h_nodelist(zidx, 4) = nidx + edgeNodes * edgeNodes; + h_nodelist(zidx, 5) = nidx + edgeNodes * edgeNodes + 1; + h_nodelist(zidx, 6) = nidx + edgeNodes * edgeNodes + edgeNodes + 1; + h_nodelist(zidx, 7) = nidx + edgeNodes * edgeNodes + edgeNodes; ++zidx; ++nidx; } @@ -366,6 +367,7 @@ Domain::BuildMesh(Int_t nx, Int_t edgeNodes, Int_t edgeElems) } nidx += edgeNodes; } + Kokkos::deep_copy(m_nodelist, h_nodelist); } //////////////////////////////////////////////////////////////////////////////// @@ -373,32 +375,31 @@ void Domain::SetupThreadSupportStructures() { // set up node-centered indexing of elements - Index_t* nodeElemCount = Allocate(numNode()); - - for(Index_t i = 0; i < numNode(); ++i) - { - nodeElemCount[i] = 0; - } + Kokkos::View nodeElemCount("nodeElemCount", numNode()); + auto h_nodelist = Kokkos::create_mirror_view(m_nodelist); + Kokkos::deep_copy(h_nodelist, m_nodelist); for(Index_t i = 0; i < numElem(); ++i) { - Index_t* nl = nodelist(i); for(Index_t j = 0; j < 8; ++j) { - ++(nodeElemCount[nl[j]]); + ++(nodeElemCount[h_nodelist(i, j)]); } } - m_nodeElemStart = Allocate(numNode() + 1); + m_nodeElemStart = Kokkos::View("m_nodeElemStart", numNode() + 1); + auto h_nodeElemStart = Kokkos::create_mirror_view(m_nodeElemStart); - m_nodeElemStart[0] = 0; + h_nodeElemStart[0] = 0; for(Index_t i = 1; i <= numNode(); ++i) { - m_nodeElemStart[i] = m_nodeElemStart[i - 1] + nodeElemCount[i - 1]; + h_nodeElemStart[i] = h_nodeElemStart[i - 1] + nodeElemCount[i - 1]; } - m_nodeElemCornerList = Allocate(m_nodeElemStart[numNode()]); + m_nodeElemCornerList = + Kokkos::View("nodeElemCornerList", h_nodeElemStart[numNode()]); + auto h_nodeElemCornerList = Kokkos::create_mirror_view(m_nodeElemCornerList); for(Index_t i = 0; i < numNode(); ++i) { @@ -407,21 +408,20 @@ Domain::SetupThreadSupportStructures() for(Index_t i = 0; i < numElem(); ++i) { - Index_t* nl = nodelist(i); for(Index_t j = 0; j < 8; ++j) { - Index_t m = nl[j]; + Index_t m = h_nodelist(i, j); Index_t k = i * 8 + j; - Index_t offset = m_nodeElemStart[m] + nodeElemCount[m]; - m_nodeElemCornerList[offset] = k; + Index_t offset = h_nodeElemStart[m] + nodeElemCount[m]; + h_nodeElemCornerList[offset] = k; ++(nodeElemCount[m]); } } - Index_t clSize = m_nodeElemStart[numNode()]; + Index_t clSize = h_nodeElemStart[numNode()]; for(Index_t i = 0; i < clSize; ++i) { - Index_t clv = m_nodeElemCornerList[i]; + Index_t clv = h_nodeElemCornerList[i]; if((clv < 0) || (clv > numElem() * 8)) { fprintf( @@ -434,8 +434,8 @@ Domain::SetupThreadSupportStructures() #endif } } - - Release(&nodeElemCount); + Kokkos::deep_copy(m_nodeElemCornerList, h_nodeElemCornerList); + Kokkos::deep_copy(m_nodeElemStart, h_nodeElemStart); } //////////////////////////////////////////////////////////////////////////////// @@ -486,12 +486,9 @@ Domain::SetupCommBuffers(Int_t edgeNodes) #endif // Boundary nodesets - if(m_colLoc == 0) - m_symmX.resize(edgeNodes * edgeNodes); - if(m_rowLoc == 0) - m_symmY.resize(edgeNodes * edgeNodes); - if(m_planeLoc == 0) - m_symmZ.resize(edgeNodes * edgeNodes); + if(m_colLoc == 0) Kokkos::resize(m_symmX, edgeNodes * edgeNodes); + if(m_rowLoc == 0) Kokkos::resize(m_symmY, edgeNodes * edgeNodes); + if(m_planeLoc == 0) Kokkos::resize(m_symmZ, edgeNodes * edgeNodes); } //////////////////////////////////////////////////////////////////////////////// @@ -506,9 +503,14 @@ Domain::CreateRegionIndexSets(Int_t nr, Int_t balance) srand(0); Index_t myRank = 0; #endif - this->numReg() = nr; - m_regElemSize = Allocate(numReg()); - m_regElemlist = Allocate(numReg()); + this->numReg() = nr; + m_regElemSize = Allocate(numReg()); + auto row_map = Kokkos::View("regElemlist::row_map", numReg() + 1); + auto h_row_map = Kokkos::create_mirror_view(row_map); + auto entries = Kokkos::View("regElemlist::entries", numElem()); + m_regElemlist = t_regElemlist(entries, row_map); + auto h_regElemlist = typename t_regElemlist::HostMirror( + Kokkos::create_mirror_view(m_regElemlist.entries), h_row_map); Index_t nextIndex = 0; // if we only have one region just fill it // Fill out the regNumList with material numbers, which are always @@ -525,14 +527,14 @@ Domain::CreateRegionIndexSets(Int_t nr, Int_t balance) // If we have more than one region distribute the elements. else { - Int_t regionNum; - Int_t regionVar; - Int_t lastReg = -1; - Int_t binSize; - Index_t elements; - Index_t runto = 0; - Int_t costDenominator = 0; - Int_t* regBinEnd = Allocate(numReg()); + Int_t regionNum; + Int_t regionVar; + Int_t lastReg = -1; + Int_t binSize; + Index_t elements; + Index_t runto = 0; + Int_t costDenominator = 0; + Kokkos::View regBinEnd("regBinEnd", numReg()); // Determine the relative weights of all the regions. This is based off the -b // flag. Balance is the value passed into b. for(Index_t i = 0; i < numReg(); ++i) @@ -612,23 +614,28 @@ Domain::CreateRegionIndexSets(Int_t nr, Int_t balance) // Second, allocate each region index set for(Index_t i = 0; i < numReg(); ++i) { - m_regElemlist[i] = Allocate(regElemSize(i)); + h_row_map(i + 1) = regElemSize(i); regElemSize(i) = 0; } // Third, fill index sets for(Index_t i = 0; i < numElem(); ++i) { - Index_t r = regNumList(i) - 1; // region index == regnum-1 - Index_t regndx = regElemSize(r)++; // Note increment - regElemlist(r, regndx) = i; + Index_t r = regNumList(i) - 1; // region index == regnum-1 + Index_t regndx = regElemSize(r)++; // Note increment + h_regElemlist.entries(h_row_map(r) + regndx) = i; } + Kokkos::deep_copy(m_regElemlist.entries, h_regElemlist.entries); + Kokkos::deep_copy(row_map, h_row_map); } ///////////////////////////////////////////////////////////// void Domain::SetupSymmetryPlanes(Int_t edgeNodes) { - Index_t nidx = 0; + Index_t nidx = 0; + auto h_symmZ = Kokkos::create_mirror_view(m_symmZ); + auto h_symmY = Kokkos::create_mirror_view(m_symmY); + auto h_symmX = Kokkos::create_mirror_view(m_symmX); for(Index_t i = 0; i < edgeNodes; ++i) { Index_t planeInc = i * edgeNodes * edgeNodes; @@ -637,55 +644,72 @@ Domain::SetupSymmetryPlanes(Int_t edgeNodes) { if(m_planeLoc == 0) { - m_symmZ[nidx] = rowInc + j; + h_symmZ[nidx] = rowInc + j; } if(m_rowLoc == 0) { - m_symmY[nidx] = planeInc + j; + h_symmY[nidx] = planeInc + j; } if(m_colLoc == 0) { - m_symmX[nidx] = planeInc + j * edgeNodes; + h_symmX[nidx] = planeInc + j * edgeNodes; } ++nidx; } } + Kokkos::deep_copy(m_symmZ, h_symmZ); + Kokkos::deep_copy(m_symmY, h_symmY); + Kokkos::deep_copy(m_symmX, h_symmX); } ///////////////////////////////////////////////////////////// void Domain::SetupElementConnectivities(Int_t edgeElems) { - lxim(0) = 0; + // With C++17 we wouldn't need to do this and could run this on the GPU + // using class lambdas + auto h_lxim = Kokkos::create_mirror_view(m_lxim); + auto h_lxip = Kokkos::create_mirror_view(m_lxip); + h_lxim(0) = 0; for(Index_t i = 1; i < numElem(); ++i) { - lxim(i) = i - 1; - lxip(i - 1) = i; + h_lxim(i) = i - 1; + h_lxip(i - 1) = i; } - lxip(numElem() - 1) = numElem() - 1; + h_lxip(numElem() - 1) = numElem() - 1; + Kokkos::deep_copy(m_lxim, h_lxim); + Kokkos::deep_copy(m_lxip, h_lxip); + auto h_letam = Kokkos::create_mirror_view(m_letam); + auto h_letap = Kokkos::create_mirror_view(m_letap); for(Index_t i = 0; i < edgeElems; ++i) { - letam(i) = i; - letap(numElem() - edgeElems + i) = numElem() - edgeElems + i; + h_letam(i) = i; + h_letap(numElem() - edgeElems + i) = numElem() - edgeElems + i; } for(Index_t i = edgeElems; i < numElem(); ++i) { - letam(i) = i - edgeElems; - letap(i - edgeElems) = i; + h_letam(i) = i - edgeElems; + h_letap(i - edgeElems) = i; } + Kokkos::deep_copy(m_letam, h_letam); + Kokkos::deep_copy(m_letap, h_letap); + auto h_lzetam = Kokkos::create_mirror_view(m_lzetam); + auto h_lzetap = Kokkos::create_mirror_view(m_lzetap); for(Index_t i = 0; i < edgeElems * edgeElems; ++i) { - lzetam(i) = i; - lzetap(numElem() - edgeElems * edgeElems + i) = + h_lzetam(i) = i; + h_lzetap(numElem() - edgeElems * edgeElems + i) = numElem() - edgeElems * edgeElems + i; } for(Index_t i = edgeElems * edgeElems; i < numElem(); ++i) { - lzetam(i) = i - edgeElems * edgeElems; - lzetap(i - edgeElems * edgeElems) = i; + h_lzetam(i) = i - edgeElems * edgeElems; + h_lzetap(i - edgeElems * edgeElems) = i; } + Kokkos::deep_copy(m_lzetam, h_lzetam); + Kokkos::deep_copy(m_lzetap, h_lzetap); } ///////////////////////////////////////////////////////////// @@ -693,11 +717,24 @@ void Domain::SetupBoundaryConditions(Int_t edgeElems) { Index_t ghostIdx[6]; // offsets to ghost locations + auto h_elemBC = Kokkos::create_mirror_view(m_elemBC); + auto h_lzetam = Kokkos::create_mirror_view(m_lzetam); + auto h_lzetap = Kokkos::create_mirror_view(m_lzetap); + auto h_letam = Kokkos::create_mirror_view(m_letam); + auto h_letap = Kokkos::create_mirror_view(m_letap); + auto h_lxim = Kokkos::create_mirror_view(m_lxim); + auto h_lxip = Kokkos::create_mirror_view(m_lxip); + Kokkos::deep_copy(h_lzetam, m_lzetam); + Kokkos::deep_copy(h_lzetap, m_lzetap); + Kokkos::deep_copy(h_letam, m_letam); + Kokkos::deep_copy(h_letap, m_letap); + Kokkos::deep_copy(h_lxim, m_lxim); + Kokkos::deep_copy(h_lxip, m_lxip); // set up boundary condition information for(Index_t i = 0; i < numElem(); ++i) { - elemBC(i) = Int_t(0); + h_elemBC(i) = Int_t(0); } for(Index_t i = 0; i < 6; ++i) @@ -750,67 +787,75 @@ Domain::SetupBoundaryConditions(Int_t edgeElems) { if(m_planeLoc == 0) { - elemBC(rowInc + j) |= ZETA_M_SYMM; + h_elemBC(rowInc + j) |= ZETA_M_SYMM; } else { - elemBC(rowInc + j) |= ZETA_M_COMM; - lzetam(rowInc + j) = ghostIdx[0] + rowInc + j; + h_elemBC(rowInc + j) |= ZETA_M_COMM; + h_lzetam(rowInc + j) = ghostIdx[0] + rowInc + j; } if(m_planeLoc == m_tp - 1) { - elemBC(rowInc + j + numElem() - edgeElems * edgeElems) |= ZETA_P_FREE; + h_elemBC(rowInc + j + numElem() - edgeElems * edgeElems) |= ZETA_P_FREE; } else { - elemBC(rowInc + j + numElem() - edgeElems * edgeElems) |= ZETA_P_COMM; - lzetap(rowInc + j + numElem() - edgeElems * edgeElems) = + h_elemBC(rowInc + j + numElem() - edgeElems * edgeElems) |= ZETA_P_COMM; + h_lzetap(rowInc + j + numElem() - edgeElems * edgeElems) = ghostIdx[1] + rowInc + j; } if(m_rowLoc == 0) { - elemBC(planeInc + j) |= ETA_M_SYMM; + h_elemBC(planeInc + j) |= ETA_M_SYMM; } else { - elemBC(planeInc + j) |= ETA_M_COMM; - letam(planeInc + j) = ghostIdx[2] + rowInc + j; + h_elemBC(planeInc + j) |= ETA_M_COMM; + h_letam(planeInc + j) = ghostIdx[2] + rowInc + j; } if(m_rowLoc == m_tp - 1) { - elemBC(planeInc + j + edgeElems * edgeElems - edgeElems) |= ETA_P_FREE; + h_elemBC(planeInc + j + edgeElems * edgeElems - edgeElems) |= ETA_P_FREE; } else { - elemBC(planeInc + j + edgeElems * edgeElems - edgeElems) |= ETA_P_COMM; - letap(planeInc + j + edgeElems * edgeElems - edgeElems) = + h_elemBC(planeInc + j + edgeElems * edgeElems - edgeElems) |= ETA_P_COMM; + h_letap(planeInc + j + edgeElems * edgeElems - edgeElems) = ghostIdx[3] + rowInc + j; } if(m_colLoc == 0) { - elemBC(planeInc + j * edgeElems) |= XI_M_SYMM; + h_elemBC(planeInc + j * edgeElems) |= XI_M_SYMM; } else { - elemBC(planeInc + j * edgeElems) |= XI_M_COMM; - lxim(planeInc + j * edgeElems) = ghostIdx[4] + rowInc + j; + h_elemBC(planeInc + j * edgeElems) |= XI_M_COMM; + h_lxim(planeInc + j * edgeElems) = ghostIdx[4] + rowInc + j; } if(m_colLoc == m_tp - 1) { - elemBC(planeInc + j * edgeElems + edgeElems - 1) |= XI_P_FREE; + h_elemBC(planeInc + j * edgeElems + edgeElems - 1) |= XI_P_FREE; } else { - elemBC(planeInc + j * edgeElems + edgeElems - 1) |= XI_P_COMM; - lxip(planeInc + j * edgeElems + edgeElems - 1) = ghostIdx[5] + rowInc + j; + h_elemBC(planeInc + j * edgeElems + edgeElems - 1) |= XI_P_COMM; + h_lxip(planeInc + j * edgeElems + edgeElems - 1) = + ghostIdx[5] + rowInc + j; } } } + Kokkos::deep_copy(m_elemBC, h_elemBC); + Kokkos::deep_copy(m_lzetam, h_lzetam); + Kokkos::deep_copy(m_lzetap, h_lzetap); + Kokkos::deep_copy(m_letam, h_letam); + Kokkos::deep_copy(m_letap, h_letap); + Kokkos::deep_copy(m_lxim, h_lxim); + Kokkos::deep_copy(m_lxip, h_lxip); } /////////////////////////////////////////////////////////////////////////// diff --git a/projects/rocprofiler-systems/examples/lulesh/lulesh-util.cc b/projects/rocprofiler-systems/examples/lulesh/lulesh-util.cc index 32bb71e5fd..a8856429c6 100644 --- a/projects/rocprofiler-systems/examples/lulesh/lulesh-util.cc +++ b/projects/rocprofiler-systems/examples/lulesh/lulesh-util.cc @@ -15,8 +15,7 @@ StrToInt(const char* token, int* retVal) char* endptr; const int decimal_base = 10; - if(token == NULL) - return 0; + if(token == NULL) return 0; c = token; *retVal = (int) strtol(c, &endptr, decimal_base); @@ -229,11 +228,13 @@ VerifyAndWriteFinalOutput(Real_t elapsed_time, Domain& locDom, Int_t nx, Int_t n ((elapsed_time * 1e6) / locDom.cycle()) / (nx * nx * nx * numRanks); Index_t ElemId = 0; + auto h_e = Kokkos::create_mirror_view(locDom.e_view()); + Kokkos::deep_copy(h_e, locDom.e_view()); printf("Run completed: \n"); printf(" Problem size = %i \n", nx); printf(" MPI tasks = %i \n", numRanks); printf(" Iteration count = %i \n", locDom.cycle()); - printf(" Final Origin Energy = %12.6e \n", locDom.e(ElemId)); + printf(" Final Origin Energy = %12.6e \n", h_e(ElemId)); Real_t MaxAbsDiff = Real_t(0.0); Real_t TotalAbsDiff = Real_t(0.0); @@ -243,16 +244,14 @@ VerifyAndWriteFinalOutput(Real_t elapsed_time, Domain& locDom, Int_t nx, Int_t n { for(Index_t k = j + 1; k < nx; ++k) { - Real_t AbsDiff = FABS(locDom.e(j * nx + k) - locDom.e(k * nx + j)); + Real_t AbsDiff = FABS(h_e(j * nx + k) - h_e(k * nx + j)); TotalAbsDiff += AbsDiff; - if(MaxAbsDiff < AbsDiff) - MaxAbsDiff = AbsDiff; + if(MaxAbsDiff < AbsDiff) MaxAbsDiff = AbsDiff; - Real_t RelDiff = AbsDiff / locDom.e(k * nx + j); + Real_t RelDiff = AbsDiff / h_e(k * nx + j); - if(MaxRelDiff < RelDiff) - MaxRelDiff = RelDiff; + if(MaxRelDiff < RelDiff) MaxRelDiff = RelDiff; } } diff --git a/projects/rocprofiler-systems/examples/lulesh/lulesh-viz.cc b/projects/rocprofiler-systems/examples/lulesh/lulesh-viz.cc index c0499f926f..273435c396 100644 --- a/projects/rocprofiler-systems/examples/lulesh/lulesh-viz.cc +++ b/projects/rocprofiler-systems/examples/lulesh/lulesh-viz.cc @@ -403,8 +403,7 @@ static void LULESH_PMPIO_Close(void* file, void* udata) { DBfile* db = (DBfile*) file; - if(db) - DBClose(db); + if(db) DBClose(db); } # endif diff --git a/projects/rocprofiler-systems/examples/lulesh/lulesh.cc b/projects/rocprofiler-systems/examples/lulesh/lulesh.cc index ae35ef70a5..e62747cedc 100644 --- a/projects/rocprofiler-systems/examples/lulesh/lulesh.cc +++ b/projects/rocprofiler-systems/examples/lulesh/lulesh.cc @@ -13,10 +13,10 @@ #include "lulesh.h" -static Real_t* buffer; -static size_t buffer_size; -static size_t buffer_offset; -static int do_atomic; +static Kokkos::View buffer; +static size_t buffer_size; +static size_t buffer_offset; +static int do_atomic; void ResizeBuffer(const size_t size) @@ -25,8 +25,7 @@ ResizeBuffer(const size_t size) if(size / sizeof(Real_t) + 1 > buffer_size) { buffer_size = size / sizeof(Real_t) + 1; - Release(&buffer); - buffer = Allocate(buffer_size); + buffer = Kokkos::View("Buffer", buffer_size); } } @@ -35,7 +34,7 @@ Type* AllocateFromBuffer(const Index_t& count) { const Index_t offset = (count * sizeof(Type) + sizeof(Real_t) - 1) / sizeof(Real_t); - Real_t* ptr = buffer + buffer_offset; + Real_t* ptr = buffer.data() + buffer_offset; buffer_offset += ((offset + 511) / 512) * 512; return static_cast(ptr); } @@ -150,13 +149,13 @@ static inline void InitStressTermsForElems(Domain& domain, Real_t* sigxx, Real_t* sigyy, Real_t* sigzz, Index_t numElem) { - Kokkos::parallel_for("InitStressTermsForElems", numElem, - KOKKOS_LAMBDA(const Index_t& i) { - sigxx[i] = sigyy[i] = sigzz[i] = -domain.p(i) - domain.q(i); - }); + Kokkos::parallel_for( + "InitStressTermsForElems", numElem, KOKKOS_LAMBDA(const Index_t& i) { + sigxx[i] = sigyy[i] = sigzz[i] = -domain.p(i) - domain.q(i); + }); } -static inline void +KOKKOS_INLINE_FUNCTION void CalcElemShapeFunctionDerivatives(Real_t const x[], Real_t const y[], Real_t const z[], Real_t b[][8], Real_t* const volume) { @@ -343,7 +342,7 @@ IntegrateStressForElems(Domain& domain, Real_t* sigxx, Real_t* sigyy, Real_t* si Kokkos::parallel_for( "IntegrateStressForElems A", numElem, KOKKOS_LAMBDA(const int k) { - const Index_t* const elemToNode = domain.nodelist(k); + const Index_t* const elemToNode = &domain.nodelist(k, 0); Real_t B[3][8]; Real_t x_local[8]; Real_t y_local[8]; @@ -359,12 +358,9 @@ IntegrateStressForElems(Domain& domain, Real_t* sigxx, Real_t* sigyy, Real_t* si &fy_elem[k * 8], &fz_elem[k * 8]); }); -#ifdef KOKKOS_HAVE_CUDA - int team_size = - std::is_same::value ? 128 : 1; -#else int team_size = 1; -#endif + if(Kokkos::DefaultExecutionSpace().concurrency() > 1024) team_size = 128; + Kokkos::parallel_for( "IntegrateStressForElems B", Kokkos::TeamPolicy<>((numNode + 127) / 128, team_size, 2), @@ -398,7 +394,7 @@ IntegrateStressForElems(Domain& domain, Real_t* sigxx, Real_t* sigyy, Real_t* si }); } -static inline void +KOKKOS_INLINE_FUNCTION void VoluDer(const Real_t x0, const Real_t x1, const Real_t x2, const Real_t x3, const Real_t x4, const Real_t x5, const Real_t y0, const Real_t y1, const Real_t y2, const Real_t y3, const Real_t y4, const Real_t y5, @@ -542,7 +538,7 @@ CalcFBHourglassForceForElems( Real_t hourgam[4][8]; Real_t xd1[8]; - const Index_t* elemToNode = domain.nodelist(i2); + const Index_t* elemToNode = &domain.nodelist(i2, 0); Index_t i3 = 8 * i2; Real_t volinv = Real_t(1.0) / determ[i2]; @@ -690,12 +686,9 @@ CalcFBHourglassForceForElems( if(!do_atomic) { -#ifdef KOKKOS_HAVE_CUDA - int team_size = - std::is_same::value ? 128 : 1; -#else int team_size = 1; -#endif + if(Kokkos::DefaultExecutionSpace().concurrency() > 1024) team_size = 128; + Kokkos::parallel_for( "CalcFBHourglassForceForElems B", Kokkos::TeamPolicy<>((numNode + 127) / 128, team_size, 2), @@ -762,7 +755,7 @@ CalcHourglassControlForElems(Domain& domain, Real_t determ[], Real_t hgcoef) KOKKOS_LAMBDA(const int i, int& err) { Real_t x1[8], y1[8], z1[8]; - Index_t* elemToNode = domain.nodelist(i); + Index_t* elemToNode = &domain.nodelist(i, 0); CollectDomainNodesToElemNodes(domain, elemToNode, x1, y1, z1); CalcElemVolumeDerivative(i, v_dvdx, v_dvdy, v_dvdz, x1, y1, z1); @@ -805,27 +798,29 @@ CalcVolumeForceForElems(Domain& domain) Index_t numElem = domain.numElem(); if(numElem != 0) { - Real_t hgcoef = domain.hgcoef(); - Real_t* sigxx = Allocate(numElem); - Real_t* sigyy = Allocate(numElem); - Real_t* sigzz = Allocate(numElem); - Real_t* determ = Allocate(numElem); + Real_t hgcoef = domain.hgcoef(); + Kokkos::View sigxx("sigxx", numElem); + Kokkos::View sigyy("sigyy", numElem); + Kokkos::View sigzz("sigzz", numElem); + Kokkos::View determ("determ", numElem); - InitStressTermsForElems(domain, sigxx, sigyy, sigzz, numElem); + InitStressTermsForElems(domain, sigxx.data(), sigyy.data(), sigzz.data(), + numElem); - IntegrateStressForElems(domain, sigxx, sigyy, sigzz, determ, numElem, - domain.numNode()); + IntegrateStressForElems(domain, sigxx.data(), sigyy.data(), sigzz.data(), + determ.data(), numElem, domain.numNode()); // check for negative element volume int error = 0; - Kokkos::parallel_reduce(numElem, - KOKKOS_LAMBDA(const int k, int& err) { - if(determ[k] <= Real_t(0.0)) - { - err++; - } - }, - error); + Kokkos::parallel_reduce( + numElem, + KOKKOS_LAMBDA(const int k, int& err) { + if(determ[k] <= Real_t(0.0)) + { + err++; + } + }, + error); if(error) #if USE_MPI @@ -834,12 +829,7 @@ CalcVolumeForceForElems(Domain& domain) exit(VolumeError); #endif - CalcHourglassControlForElems(domain, determ, hgcoef); - - Release(&determ); - Release(&sigzz); - Release(&sigyy); - Release(&sigxx); + CalcHourglassControlForElems(domain, determ.data(), hgcoef); } } @@ -853,11 +843,12 @@ CalcForceForNodes(Domain& domain) domain.sizeZ() + 1, true, false); #endif - Kokkos::parallel_for("CalcForceForNodes", numNode, KOKKOS_LAMBDA(const int i) { - domain.fx(i) = Real_t(0.0); - domain.fy(i) = Real_t(0.0); - domain.fz(i) = Real_t(0.0); - }); + Kokkos::parallel_for( + "CalcForceForNodes", numNode, KOKKOS_LAMBDA(const int i) { + domain.fx(i) = Real_t(0.0); + domain.fy(i) = Real_t(0.0); + domain.fz(i) = Real_t(0.0); + }); CalcVolumeForceForElems(domain); @@ -876,11 +867,12 @@ CalcForceForNodes(Domain& domain) static inline void CalcAccelerationForNodes(Domain& domain, Index_t numNode) { - Kokkos::parallel_for("CalcAccelerationForNodes", numNode, KOKKOS_LAMBDA(const int i) { - domain.xdd(i) = domain.fx(i) / domain.nodalMass(i); - domain.ydd(i) = domain.fy(i) / domain.nodalMass(i); - domain.zdd(i) = domain.fz(i) / domain.nodalMass(i); - }); + Kokkos::parallel_for( + "CalcAccelerationForNodes", numNode, KOKKOS_LAMBDA(const int i) { + domain.xdd(i) = domain.fx(i) / domain.nodalMass(i); + domain.ydd(i) = domain.fy(i) / domain.nodalMass(i); + domain.zdd(i) = domain.fz(i) / domain.nodalMass(i); + }); } static inline void @@ -914,34 +906,33 @@ ApplyAccelerationBoundaryConditionsForNodes(Domain& domain) static inline void CalcVelocityForNodes(Domain& domain, const Real_t dt, const Real_t u_cut, Index_t numNode) { - Kokkos::parallel_for("CalcVelocityForNodes", numNode, KOKKOS_LAMBDA(const int i) { - Real_t xdtmp, ydtmp, zdtmp; + Kokkos::parallel_for( + "CalcVelocityForNodes", numNode, KOKKOS_LAMBDA(const int i) { + Real_t xdtmp, ydtmp, zdtmp; - xdtmp = domain.xd(i) + domain.xdd(i) * dt; - if(FABS(xdtmp) < u_cut) - xdtmp = Real_t(0.0); - domain.xd(i) = xdtmp; + xdtmp = domain.xd(i) + domain.xdd(i) * dt; + if(FABS(xdtmp) < u_cut) xdtmp = Real_t(0.0); + domain.xd(i) = xdtmp; - ydtmp = domain.yd(i) + domain.ydd(i) * dt; - if(FABS(ydtmp) < u_cut) - ydtmp = Real_t(0.0); - domain.yd(i) = ydtmp; + ydtmp = domain.yd(i) + domain.ydd(i) * dt; + if(FABS(ydtmp) < u_cut) ydtmp = Real_t(0.0); + domain.yd(i) = ydtmp; - zdtmp = domain.zd(i) + domain.zdd(i) * dt; - if(FABS(zdtmp) < u_cut) - zdtmp = Real_t(0.0); - domain.zd(i) = zdtmp; - }); + zdtmp = domain.zd(i) + domain.zdd(i) * dt; + if(FABS(zdtmp) < u_cut) zdtmp = Real_t(0.0); + domain.zd(i) = zdtmp; + }); } static inline void CalcPositionForNodes(Domain& domain, const Real_t dt, Index_t numNode) { - Kokkos::parallel_for("CalcPositionForNodes", numNode, KOKKOS_LAMBDA(const int i) { - domain.x(i) += domain.xd(i) * dt; - domain.y(i) += domain.yd(i) * dt; - domain.z(i) += domain.zd(i) * dt; - }); + Kokkos::parallel_for( + "CalcPositionForNodes", numNode, KOKKOS_LAMBDA(const int i) { + domain.x(i) += domain.xd(i) * dt; + domain.y(i) += domain.yd(i) * dt; + domain.z(i) += domain.zd(i) * dt; + }); } static inline void @@ -989,12 +980,12 @@ LagrangeNodal(Domain& domain) } KOKKOS_INLINE_FUNCTION Real_t - CalcElemVolume(const Real_t x0, const Real_t x1, const Real_t x2, const Real_t x3, - const Real_t x4, const Real_t x5, const Real_t x6, const Real_t x7, - const Real_t y0, const Real_t y1, const Real_t y2, const Real_t y3, - const Real_t y4, const Real_t y5, const Real_t y6, const Real_t y7, - const Real_t z0, const Real_t z1, const Real_t z2, const Real_t z3, - const Real_t z4, const Real_t z5, const Real_t z6, const Real_t z7) +CalcElemVolume(const Real_t x0, const Real_t x1, const Real_t x2, const Real_t x3, + const Real_t x4, const Real_t x5, const Real_t x6, const Real_t x7, + const Real_t y0, const Real_t y1, const Real_t y2, const Real_t y3, + const Real_t y4, const Real_t y5, const Real_t y6, const Real_t y7, + const Real_t z0, const Real_t z1, const Real_t z2, const Real_t z3, + const Real_t z4, const Real_t z5, const Real_t z6, const Real_t z7) { Real_t twelveth = Real_t(1.0) / Real_t(12.0); @@ -1073,7 +1064,8 @@ CalcElemVolume(const Real_t x[8], const Real_t y[8], const Real_t z[8]) z[4], z[5], z[6], z[7]); } -static inline Real_t +KOKKOS_INLINE_FUNCTION +Real_t AreaFace(const Real_t x0, const Real_t x1, const Real_t x2, const Real_t x3, const Real_t y0, const Real_t y1, const Real_t y2, const Real_t y3, const Real_t z0, const Real_t z1, const Real_t z2, const Real_t z3) @@ -1090,8 +1082,8 @@ AreaFace(const Real_t x0, const Real_t x1, const Real_t x2, const Real_t x3, } KOKKOS_INLINE_FUNCTION Real_t - CalcElemCharacteristicLength(const Real_t x[8], const Real_t y[8], const Real_t z[8], - const Real_t volume) +CalcElemCharacteristicLength(const Real_t x[8], const Real_t y[8], const Real_t z[8], + const Real_t volume) { Real_t a, charLength = Real_t(0.0); @@ -1118,7 +1110,7 @@ KOKKOS_INLINE_FUNCTION Real_t return charLength; } -static inline void +KOKKOS_INLINE_FUNCTION void CalcElemVelocityGradient(const Real_t* const xvel, const Real_t* const yvel, const Real_t* const zvel, const Real_t b[][8], const Real_t detJ, Real_t* const d) @@ -1163,55 +1155,56 @@ CalcElemVelocityGradient(const Real_t* const xvel, const Real_t* const yvel, void CalcKinematicsForElems(Domain& domain, Real_t deltaTime, Index_t numElem) { - Kokkos::parallel_for("CalcKinematicsForElems", numElem, KOKKOS_LAMBDA(const int k) { - Real_t B[3][8]; - Real_t D[6]; - Real_t x_local[8]; - Real_t y_local[8]; - Real_t z_local[8]; - Real_t xd_local[8]; - Real_t yd_local[8]; - Real_t zd_local[8]; - Real_t detJ = Real_t(0.0); + Kokkos::parallel_for( + "CalcKinematicsForElems", numElem, KOKKOS_LAMBDA(const int k) { + Real_t B[3][8]; + Real_t D[6]; + Real_t x_local[8]; + Real_t y_local[8]; + Real_t z_local[8]; + Real_t xd_local[8]; + Real_t yd_local[8]; + Real_t zd_local[8]; + Real_t detJ = Real_t(0.0); - Real_t volume; - Real_t relativeVolume; - const Index_t* const elemToNode = domain.nodelist(k); + Real_t volume; + Real_t relativeVolume; + const Index_t* const elemToNode = &domain.nodelist(k, 0); - CollectDomainNodesToElemNodes(domain, elemToNode, x_local, y_local, z_local); + CollectDomainNodesToElemNodes(domain, elemToNode, x_local, y_local, z_local); - volume = CalcElemVolume(x_local, y_local, z_local); - relativeVolume = volume / domain.volo(k); - domain.vnew(k) = relativeVolume; - domain.delv(k) = relativeVolume - domain.v(k); + volume = CalcElemVolume(x_local, y_local, z_local); + relativeVolume = volume / domain.volo(k); + domain.vnew(k) = relativeVolume; + domain.delv(k) = relativeVolume - domain.v(k); - domain.arealg(k) = - CalcElemCharacteristicLength(x_local, y_local, z_local, volume); + domain.arealg(k) = + CalcElemCharacteristicLength(x_local, y_local, z_local, volume); - for(Index_t lnode = 0; lnode < 8; ++lnode) - { - Index_t gnode = elemToNode[lnode]; - xd_local[lnode] = domain.c_xd(gnode); - yd_local[lnode] = domain.c_yd(gnode); - zd_local[lnode] = domain.c_zd(gnode); - } + for(Index_t lnode = 0; lnode < 8; ++lnode) + { + Index_t gnode = elemToNode[lnode]; + xd_local[lnode] = domain.c_xd(gnode); + yd_local[lnode] = domain.c_yd(gnode); + zd_local[lnode] = domain.c_zd(gnode); + } - Real_t dt2 = Real_t(0.5) * deltaTime; - for(Index_t j = 0; j < 8; ++j) - { - x_local[j] -= dt2 * xd_local[j]; - y_local[j] -= dt2 * yd_local[j]; - z_local[j] -= dt2 * zd_local[j]; - } + Real_t dt2 = Real_t(0.5) * deltaTime; + for(Index_t j = 0; j < 8; ++j) + { + x_local[j] -= dt2 * xd_local[j]; + y_local[j] -= dt2 * yd_local[j]; + z_local[j] -= dt2 * zd_local[j]; + } - CalcElemShapeFunctionDerivatives(x_local, y_local, z_local, B, &detJ); + CalcElemShapeFunctionDerivatives(x_local, y_local, z_local, B, &detJ); - CalcElemVelocityGradient(xd_local, yd_local, zd_local, B, detJ, D); + CalcElemVelocityGradient(xd_local, yd_local, zd_local, B, detJ, D); - domain.dxx(k) = D[0]; - domain.dyy(k) = D[1]; - domain.dzz(k) = D[2]; - }); + domain.dxx(k) = D[0]; + domain.dyy(k) = D[1]; + domain.dzz(k) = D[2]; + }); } static inline void @@ -1227,23 +1220,23 @@ CalcLagrangeElements(Domain& domain) CalcKinematicsForElems(domain, deltatime, numElem); int error = 0; - Kokkos::parallel_reduce(numElem, - KOKKOS_LAMBDA(const int k, int& err) { - Real_t vdov = - domain.dxx(k) + domain.dyy(k) + domain.dzz(k); - Real_t vdovthird = vdov / Real_t(3.0); + Kokkos::parallel_reduce( + numElem, + KOKKOS_LAMBDA(const int k, int& err) { + Real_t vdov = domain.dxx(k) + domain.dyy(k) + domain.dzz(k); + Real_t vdovthird = vdov / Real_t(3.0); - domain.vdov(k) = vdov; - domain.dxx(k) -= vdovthird; - domain.dyy(k) -= vdovthird; - domain.dzz(k) -= vdovthird; + domain.vdov(k) = vdov; + domain.dxx(k) -= vdovthird; + domain.dyy(k) -= vdovthird; + domain.dzz(k) -= vdovthird; - if(domain.vnew(k) <= Real_t(0.0)) - { - err++; - } - }, - error); + if(domain.vnew(k) <= Real_t(0.0)) + { + err++; + } + }, + error); if(error) #if USE_MPI @@ -1267,7 +1260,7 @@ CalcMonotonicQGradientsForElems(Domain& domain) Real_t ax, ay, az; Real_t dxv, dyv, dzv; - const Index_t* elemToNode = domain.nodelist(i); + const Index_t* elemToNode = &domain.nodelist(i, 0); Index_t n0 = elemToNode[0]; Index_t n1 = elemToNode[1]; Index_t n2 = elemToNode[2]; @@ -1446,14 +1439,10 @@ CalcMonotonicQRegionForElems(Domain& domain, Int_t r, Real_t ptiny) delvm *= monoq_limiter_mult; delvp *= monoq_limiter_mult; - if(delvm < phixi) - phixi = delvm; - if(delvp < phixi) - phixi = delvp; - if(phixi < Real_t(0.)) - phixi = Real_t(0.); - if(phixi > monoq_max_slope) - phixi = monoq_max_slope; + if(delvm < phixi) phixi = delvm; + if(delvp < phixi) phixi = delvp; + if(phixi < Real_t(0.)) phixi = Real_t(0.); + if(phixi > monoq_max_slope) phixi = monoq_max_slope; norm = Real_t(1.) / (domain.delv_eta(ielem) + ptiny); @@ -1488,14 +1477,10 @@ CalcMonotonicQRegionForElems(Domain& domain, Int_t r, Real_t ptiny) delvm *= monoq_limiter_mult; delvp *= monoq_limiter_mult; - if(delvm < phieta) - phieta = delvm; - if(delvp < phieta) - phieta = delvp; - if(phieta < Real_t(0.)) - phieta = Real_t(0.); - if(phieta > monoq_max_slope) - phieta = monoq_max_slope; + if(delvm < phieta) phieta = delvm; + if(delvp < phieta) phieta = delvp; + if(phieta < Real_t(0.)) phieta = Real_t(0.); + if(phieta > monoq_max_slope) phieta = monoq_max_slope; norm = Real_t(1.) / (domain.delv_zeta(ielem) + ptiny); @@ -1530,14 +1515,10 @@ CalcMonotonicQRegionForElems(Domain& domain, Int_t r, Real_t ptiny) delvm *= monoq_limiter_mult; delvp *= monoq_limiter_mult; - if(delvm < phizeta) - phizeta = delvm; - if(delvp < phizeta) - phizeta = delvp; - if(phizeta < Real_t(0.)) - phizeta = Real_t(0.); - if(phizeta > monoq_max_slope) - phizeta = monoq_max_slope; + if(delvm < phizeta) phizeta = delvm; + if(delvp < phizeta) phizeta = delvp; + if(phizeta < Real_t(0.)) phizeta = Real_t(0.); + if(phizeta > monoq_max_slope) phizeta = monoq_max_slope; if(domain.vdov(ielem) > Real_t(0.)) { @@ -1550,12 +1531,9 @@ CalcMonotonicQRegionForElems(Domain& domain, Int_t r, Real_t ptiny) Real_t delvxeta = domain.delv_eta(ielem) * domain.delx_eta(ielem); Real_t delvxzeta = domain.delv_zeta(ielem) * domain.delx_zeta(ielem); - if(delvxxi > Real_t(0.)) - delvxxi = Real_t(0.); - if(delvxeta > Real_t(0.)) - delvxeta = Real_t(0.); - if(delvxzeta > Real_t(0.)) - delvxzeta = Real_t(0.); + if(delvxxi > Real_t(0.)) delvxxi = Real_t(0.); + if(delvxeta > Real_t(0.)) delvxeta = Real_t(0.); + if(delvxzeta > Real_t(0.)) delvxzeta = Real_t(0.); Real_t rho = domain.elemMass(ielem) / (domain.volo(ielem) * domain.vnew(ielem)); @@ -1628,14 +1606,15 @@ CalcQForElems(Domain& domain) domain.DeallocateGradients(); Index_t idx = 0; - Kokkos::parallel_reduce(numElem, - KOKKOS_LAMBDA(const Index_t& i, Index_t& count) { - if(domain.q(i) > domain.qstop()) - { - count++; - } - }, - idx); + Kokkos::parallel_reduce( + numElem, + KOKKOS_LAMBDA(const Index_t& i, Index_t& count) { + if(domain.q(i) > domain.qstop()) + { + count++; + } + }, + idx); if(idx > 0) { @@ -1661,14 +1640,12 @@ CalcPressureForElem(Real_t& p_new_i, Real_t& bvc_i, Real_t& pbvc_i, const Real_t p_new_i = bvc_i * e_old_i; - if(FABS(p_new_i) < p_cut) - p_new_i = Real_t(0.0); + if(FABS(p_new_i) < p_cut) p_new_i = Real_t(0.0); if(vnewc_e >= eosvmax) /* impossible condition here? */ p_new_i = Real_t(0.0); - if(p_new_i < pmin) - p_new_i = pmin; + if(p_new_i < pmin) p_new_i = pmin; } static inline void @@ -1677,135 +1654,138 @@ CalcEnergyForElems(Real_t* p_new, Real_t* e_new, Real_t* q_new, Real_t* bvc, Rea Real_t* compHalfStep, Real_t* vnewc, Real_t* work, Real_t* delvc, Real_t pmin, Real_t p_cut, Real_t e_cut, Real_t q_cut, Real_t emin, Real_t* qq_old, Real_t* ql_old, Real_t rho0, Real_t eosvmax, - Index_t length, Index_t* regElemList) + Index_t length, Domain& domain, Index_t r) { - Kokkos::parallel_for("CalcEnergyForElems", length, KOKKOS_LAMBDA(const int i) { - const Real_t delvc_i = delvc[i]; - const Real_t p_old_i = p_old[i]; - const Real_t q_old_i = q_old[i]; - Real_t e_new_i = e_old[i] - Real_t(0.5) * delvc_i * (p_old_i + q_old_i) + - Real_t(0.5) * work[i]; + Kokkos::parallel_for( + "CalcEnergyForElems", length, KOKKOS_LAMBDA(const int i) { + const Real_t delvc_i = delvc[i]; + const Real_t p_old_i = p_old[i]; + const Real_t q_old_i = q_old[i]; + Real_t e_new_i = e_old[i] - Real_t(0.5) * delvc_i * (p_old_i + q_old_i) + + Real_t(0.5) * work[i]; - if(e_new_i < emin) - { - e_new_i = emin; - } - - Real_t bvc_i, pbvc_i; - Real_t pHalfStep_i; - const Real_t vnewc_e = vnewc[regElemList[i]]; - const Real_t compHalfStep_i = compHalfStep[i]; - CalcPressureForElem(pHalfStep_i, bvc_i, pbvc_i, e_new_i, compHalfStep_i, vnewc_e, - pmin, p_cut, eosvmax); - - Real_t vhalf = Real_t(1.) / (Real_t(1.) + compHalfStep_i); - - Real_t q_new_i; - const Real_t ql_old_i = ql_old[i]; - const Real_t qq_old_i = qq_old[i]; - if(delvc_i > Real_t(0.)) - { - q_new_i /* = qq_old[i] = ql_old[i] */ = Real_t(0.); - } - else - { - Real_t ssc = (pbvc_i * e_new_i + vhalf * vhalf * bvc_i * pHalfStep_i) / rho0; - - if(ssc <= Real_t(.1111111e-36)) + if(e_new_i < emin) { - ssc = Real_t(.3333333e-18); + e_new_i = emin; + } + + Real_t bvc_i, pbvc_i; + Real_t pHalfStep_i; + const Real_t vnewc_e = vnewc[domain.regElemlist(r, i)]; + const Real_t compHalfStep_i = compHalfStep[i]; + CalcPressureForElem(pHalfStep_i, bvc_i, pbvc_i, e_new_i, compHalfStep_i, + vnewc_e, pmin, p_cut, eosvmax); + + Real_t vhalf = Real_t(1.) / (Real_t(1.) + compHalfStep_i); + + Real_t q_new_i; + const Real_t ql_old_i = ql_old[i]; + const Real_t qq_old_i = qq_old[i]; + if(delvc_i > Real_t(0.)) + { + q_new_i /* = qq_old[i] = ql_old[i] */ = Real_t(0.); } else { - ssc = SQRT(ssc); + Real_t ssc = + (pbvc_i * e_new_i + vhalf * vhalf * bvc_i * pHalfStep_i) / rho0; + + if(ssc <= Real_t(.1111111e-36)) + { + ssc = Real_t(.3333333e-18); + } + else + { + ssc = SQRT(ssc); + } + + q_new_i = (ssc * ql_old_i + qq_old_i); } - q_new_i = (ssc * ql_old_i + qq_old_i); - } + e_new_i = e_new_i + Real_t(0.5) * delvc_i * + (Real_t(3.0) * (p_old_i + q_old_i) - + Real_t(4.0) * (pHalfStep_i + q_new_i)); - e_new_i = e_new_i + Real_t(0.5) * delvc_i * - (Real_t(3.0) * (p_old_i + q_old_i) - - Real_t(4.0) * (pHalfStep_i + q_new_i)); + e_new_i += Real_t(0.5) * work[i]; - e_new_i += Real_t(0.5) * work[i]; - - if(FABS(e_new_i) < e_cut) - { - e_new_i = Real_t(0.); - } - if(e_new_i < emin) - { - e_new_i = emin; - } - Real_t p_new_i; - const Real_t compression_i = compression[i]; - CalcPressureForElem(p_new_i, bvc_i, pbvc_i, e_new_i, compression_i, vnewc_e, pmin, - p_cut, eosvmax); - - const Real_t sixth = Real_t(1.0) / Real_t(6.0); - Real_t q_tilde; - - if(delvc_i > Real_t(0.)) - { - q_tilde = Real_t(0.); - } - else - { - Real_t ssc = (pbvc_i * e_new_i + vnewc_e * vnewc_e * bvc_i * p_new_i) / rho0; - - if(ssc <= Real_t(.1111111e-36)) + if(FABS(e_new_i) < e_cut) { - ssc = Real_t(.3333333e-18); + e_new_i = Real_t(0.); + } + if(e_new_i < emin) + { + e_new_i = emin; + } + Real_t p_new_i; + const Real_t compression_i = compression[i]; + CalcPressureForElem(p_new_i, bvc_i, pbvc_i, e_new_i, compression_i, vnewc_e, + pmin, p_cut, eosvmax); + + const Real_t sixth = Real_t(1.0) / Real_t(6.0); + Real_t q_tilde; + + if(delvc_i > Real_t(0.)) + { + q_tilde = Real_t(0.); } else { - ssc = SQRT(ssc); + Real_t ssc = + (pbvc_i * e_new_i + vnewc_e * vnewc_e * bvc_i * p_new_i) / rho0; + + if(ssc <= Real_t(.1111111e-36)) + { + ssc = Real_t(.3333333e-18); + } + else + { + ssc = SQRT(ssc); + } + + q_tilde = (ssc * ql_old_i + qq_old_i); } - q_tilde = (ssc * ql_old_i + qq_old_i); - } + e_new_i = + e_new_i - (Real_t(7.0) * (p_old_i + q_old_i) - + Real_t(8.0) * (pHalfStep_i + q_new_i) + (p_new_i + q_tilde)) * + delvc_i * sixth; - e_new_i = - e_new_i - (Real_t(7.0) * (p_old_i + q_old_i) - - Real_t(8.0) * (pHalfStep_i + q_new_i) + (p_new_i + q_tilde)) * - delvc_i * sixth; - - if(FABS(e_new_i) < e_cut) - { - e_new_i = Real_t(0.); - } - if(e_new_i < emin) - { - e_new_i = emin; - } - - CalcPressureForElem(p_new_i, bvc_i, pbvc_i, e_new_i, compression_i, vnewc_e, pmin, - p_cut, eosvmax); - bvc[i] = bvc_i; - pbvc[i] = pbvc_i; - p_new[i] = p_new_i; - - if(delvc_i <= Real_t(0.)) - { - Real_t ssc = (pbvc_i * e_new_i + vnewc_e * vnewc_e * bvc_i * p_new_i) / rho0; - - if(ssc <= Real_t(.1111111e-36)) + if(FABS(e_new_i) < e_cut) { - ssc = Real_t(.3333333e-18); + e_new_i = Real_t(0.); } - else + if(e_new_i < emin) { - ssc = SQRT(ssc); + e_new_i = emin; } - q_new_i = (ssc * ql_old_i + qq_old_i); + CalcPressureForElem(p_new_i, bvc_i, pbvc_i, e_new_i, compression_i, vnewc_e, + pmin, p_cut, eosvmax); + bvc[i] = bvc_i; + pbvc[i] = pbvc_i; + p_new[i] = p_new_i; - if(FABS(q_new_i) < q_cut) - q_new_i = Real_t(0.); - } - q_new[i] = q_new_i; - e_new[i] = e_new_i; - }); + if(delvc_i <= Real_t(0.)) + { + Real_t ssc = + (pbvc_i * e_new_i + vnewc_e * vnewc_e * bvc_i * p_new_i) / rho0; + + if(ssc <= Real_t(.1111111e-36)) + { + ssc = Real_t(.3333333e-18); + } + else + { + ssc = SQRT(ssc); + } + + q_new_i = (ssc * ql_old_i + qq_old_i); + + if(FABS(q_new_i) < q_cut) q_new_i = Real_t(0.); + } + q_new[i] = q_new_i; + e_new[i] = e_new_i; + }); return; } @@ -1813,27 +1793,28 @@ CalcEnergyForElems(Real_t* p_new, Real_t* e_new, Real_t* q_new, Real_t* bvc, Rea static inline void CalcSoundSpeedForElems(Domain& domain, Real_t* vnewc, Real_t rho0, Real_t* enewc, Real_t* pnewc, Real_t* pbvc, Real_t* bvc, Real_t ss4o3, - Index_t len, Index_t* regElemList) + Index_t len, Index_t r) { - Kokkos::parallel_for("CalcSoundSpeedForElems", len, KOKKOS_LAMBDA(const int i) { - Index_t ielem = regElemList[i]; - Real_t ssTmp = - (pbvc[i] * enewc[i] + vnewc[ielem] * vnewc[ielem] * bvc[i] * pnewc[i]) / rho0; - if(ssTmp <= Real_t(.1111111e-36)) - { - ssTmp = Real_t(.3333333e-18); - } - else - { - ssTmp = SQRT(ssTmp); - } - domain.ss(ielem) = ssTmp; - }); + Kokkos::parallel_for( + "CalcSoundSpeedForElems", len, KOKKOS_LAMBDA(const int i) { + Index_t ielem = domain.regElemlist(r, i); + Real_t ssTmp = + (pbvc[i] * enewc[i] + vnewc[ielem] * vnewc[ielem] * bvc[i] * pnewc[i]) / + rho0; + if(ssTmp <= Real_t(.1111111e-36)) + { + ssTmp = Real_t(.3333333e-18); + } + else + { + ssTmp = SQRT(ssTmp); + } + domain.ss(ielem) = ssTmp; + }); } static inline void -EvalEOSForElems(Domain& domain, Real_t* vnewc, Int_t numElemReg, Index_t* regElemList, - Int_t rep) +EvalEOSForElems(Domain& domain, Real_t* vnewc, Int_t numElemReg, Index_t r, Int_t rep) { Real_t e_cut = domain.e_cut(); Real_t p_cut = domain.p_cut(); @@ -1865,54 +1846,56 @@ EvalEOSForElems(Domain& domain, Real_t* vnewc, Int_t numElemReg, Index_t* regEle for(Int_t j = 0; j < rep; j++) { - Kokkos::parallel_for("EvalEOSForElems A", numElemReg, KOKKOS_LAMBDA(const int i) { - Index_t ielem = regElemList[i]; - e_old[i] = domain.c_e(ielem); - delvc[i] = domain.c_delv(ielem); - p_old[i] = domain.c_p(ielem); - q_old[i] = domain.c_q(ielem); - qq_old[i] = domain.c_qq(ielem); - ql_old[i] = domain.c_ql(ielem); - const Real_t vnewc_ielem = vnewc[ielem]; - Real_t vchalf; - compression[i] = Real_t(1.) / vnewc_ielem - Real_t(1.); - vchalf = vnewc_ielem - delvc[i] * Real_t(.5); - compHalfStep[i] = Real_t(1.) / vchalf - Real_t(1.); + Kokkos::parallel_for( + "EvalEOSForElems A", numElemReg, KOKKOS_LAMBDA(const int i) { + Index_t ielem = domain.regElemlist(r, i); + e_old[i] = domain.c_e(ielem); + delvc[i] = domain.c_delv(ielem); + p_old[i] = domain.c_p(ielem); + q_old[i] = domain.c_q(ielem); + qq_old[i] = domain.c_qq(ielem); + ql_old[i] = domain.c_ql(ielem); + const Real_t vnewc_ielem = vnewc[ielem]; + Real_t vchalf; + compression[i] = Real_t(1.) / vnewc_ielem - Real_t(1.); + vchalf = vnewc_ielem - delvc[i] * Real_t(.5); + compHalfStep[i] = Real_t(1.) / vchalf - Real_t(1.); - if(eosvmin != Real_t(0.)) - { - if(vnewc_ielem <= eosvmin) - { /* impossible due to calling func? */ - compHalfStep[i] = compression[i]; + if(eosvmin != Real_t(0.)) + { + if(vnewc_ielem <= eosvmin) + { /* impossible due to calling func? */ + compHalfStep[i] = compression[i]; + } } - } - if(eosvmax != Real_t(0.)) - { - if(vnewc_ielem >= eosvmax) - { /* impossible due to calling func? */ - p_old[i] = Real_t(0.); - compression[i] = Real_t(0.); - compHalfStep[i] = Real_t(0.); + if(eosvmax != Real_t(0.)) + { + if(vnewc_ielem >= eosvmax) + { /* impossible due to calling func? */ + p_old[i] = Real_t(0.); + compression[i] = Real_t(0.); + compHalfStep[i] = Real_t(0.); + } } - } - work[i] = Real_t(0.); - }); + work[i] = Real_t(0.); + }); CalcEnergyForElems(p_new, e_new, q_new, bvc, pbvc, p_old, e_old, q_old, compression, compHalfStep, vnewc, work, delvc, pmin, p_cut, e_cut, q_cut, emin, qq_old, ql_old, rho0, eosvmax, numElemReg, - regElemList); + domain, r); } - Kokkos::parallel_for("EvalEOSForElems F", numElemReg, KOKKOS_LAMBDA(const int i) { - Index_t ielem = regElemList[i]; - domain.p(ielem) = p_new[i]; - domain.e(ielem) = e_new[i]; - domain.q(ielem) = q_new[i]; - }); + Kokkos::parallel_for( + "EvalEOSForElems F", numElemReg, KOKKOS_LAMBDA(const int i) { + Index_t ielem = domain.regElemlist(r, i); + domain.p(ielem) = p_new[i]; + domain.e(ielem) = e_new[i]; + domain.q(ielem) = q_new[i]; + }); CalcSoundSpeedForElems(domain, vnewc, rho0, e_new, p_new, pbvc, bvc, ss4o3, - numElemReg, regElemList); + numElemReg, r); } static inline void @@ -1922,51 +1905,49 @@ ApplyMaterialPropertiesForElems(Domain& domain) if(numElem != 0) { - Real_t eosvmin = domain.eosvmin(); - Real_t eosvmax = domain.eosvmax(); - Real_t* vnewc = Allocate(numElem); + Real_t eosvmin = domain.eosvmin(); + Real_t eosvmax = domain.eosvmax(); + Kokkos::View vnewc("vnewc", numElem); - Kokkos::parallel_for("ApplyMaterialPropertiesForElems A", numElem, - KOKKOS_LAMBDA(const int i) { vnewc[i] = domain.vnew(i); }); + Kokkos::parallel_for( + "ApplyMaterialPropertiesForElems A", numElem, + KOKKOS_LAMBDA(const int i) { vnewc[i] = domain.vnew(i); }); if(eosvmin != Real_t(0.)) { - Kokkos::parallel_for("ApplyMaterialPropertiesForElems B", numElem, - KOKKOS_LAMBDA(const int i) { - if(vnewc[i] < eosvmin) - vnewc[i] = eosvmin; - }); + Kokkos::parallel_for( + "ApplyMaterialPropertiesForElems B", numElem, KOKKOS_LAMBDA(const int i) { + if(vnewc[i] < eosvmin) vnewc[i] = eosvmin; + }); } if(eosvmax != Real_t(0.)) { - Kokkos::parallel_for("ApplyMaterialPropertiesForElems C", numElem, - KOKKOS_LAMBDA(const int i) { - if(vnewc[i] > eosvmax) - vnewc[i] = eosvmax; - }); + Kokkos::parallel_for( + "ApplyMaterialPropertiesForElems C", numElem, KOKKOS_LAMBDA(const int i) { + if(vnewc[i] > eosvmax) vnewc[i] = eosvmax; + }); } int error = 0; - Kokkos::parallel_reduce(numElem, - KOKKOS_LAMBDA(const int i, int& err) { - Real_t vc = domain.v(i); - if(eosvmin != Real_t(0.)) - { - if(vc < eosvmin) - vc = eosvmin; - } - if(eosvmax != Real_t(0.)) - { - if(vc > eosvmax) - vc = eosvmax; - } - if(vc <= 0.) - { - err++; - } - }, - error); + Kokkos::parallel_reduce( + numElem, + KOKKOS_LAMBDA(const int i, int& err) { + Real_t vc = domain.v(i); + if(eosvmin != Real_t(0.)) + { + if(vc < eosvmin) vc = eosvmin; + } + if(eosvmax != Real_t(0.)) + { + if(vc > eosvmax) vc = eosvmax; + } + if(vc <= 0.) + { + err++; + } + }, + error); if(error) #if USE_MPI @@ -1977,19 +1958,17 @@ ApplyMaterialPropertiesForElems(Domain& domain) for(Int_t r = 0; r < domain.numReg(); r++) { - Index_t numElemReg = domain.regElemSize(r); - Index_t* regElemList = domain.regElemlist(r); - Int_t rep; + Index_t numElemReg = domain.regElemSize(r); + // Index_t *regElemList = domain.regElemlist(r); + Int_t rep; if(r < domain.numReg() / 2) rep = 1; else if(r < (domain.numReg() - (domain.numReg() + 15) / 20)) rep = 1 + domain.cost(); else rep = 10 * (1 + domain.cost()); - EvalEOSForElems(domain, vnewc, numElemReg, regElemList, rep); + EvalEOSForElems(domain, vnewc.data(), numElemReg, r, rep); } - - Release(&vnewc); } } @@ -1998,14 +1977,14 @@ UpdateVolumesForElems(Domain& domain, Real_t v_cut, Index_t length) { if(length != 0) { - Kokkos::parallel_for("UpdateVolumesForElems", length, KOKKOS_LAMBDA(const int i) { - Real_t tmpV = domain.vnew(i); + Kokkos::parallel_for( + "UpdateVolumesForElems", length, KOKKOS_LAMBDA(const int i) { + Real_t tmpV = domain.vnew(i); - if(FABS(tmpV - Real_t(1.0)) < v_cut) - tmpV = Real_t(1.0); + if(FABS(tmpV - Real_t(1.0)) < v_cut) tmpV = Real_t(1.0); - domain.v(i) = tmpV; - }); + domain.v(i) = tmpV; + }); } return; @@ -2024,8 +2003,8 @@ LagrangeElements(Domain& domain, Index_t numElem) } static inline void -CalcCourantConstraintForElems(Domain& domain, Index_t length, Index_t* regElemlist, - Real_t qqc, Real_t& dtcourant) +CalcCourantConstraintForElems(Domain& domain, Index_t length, Index_t r, Real_t qqc, + Real_t& dtcourant) { typedef Kokkos::View view_real_t; @@ -2035,28 +2014,28 @@ CalcCourantConstraintForElems(Domain& domain, Index_t length, Index_t* regElemli MinFinder result; - Kokkos::parallel_reduce(length, - KOKKOS_LAMBDA(const int i, MinFinder& minf) { - Index_t indx = regElemlist[i]; - Real_t dtf = domain.ss(indx) * domain.ss(indx); + Kokkos::parallel_reduce( + length, + KOKKOS_LAMBDA(const int i, MinFinder& minf) { + Index_t indx = domain.regElemlist(r, i); + Real_t dtf = domain.ss(indx) * domain.ss(indx); - if(domain.vdov(indx) < Real_t(0.)) - { - dtf = dtf + qqc2 * domain.arealg(indx) * - domain.arealg(indx) * - domain.vdov(indx) * domain.vdov(indx); - } + if(domain.vdov(indx) < Real_t(0.)) + { + dtf = dtf + qqc2 * domain.arealg(indx) * domain.arealg(indx) * + domain.vdov(indx) * domain.vdov(indx); + } - dtf = SQRT(dtf); - dtf = domain.arealg(indx) / dtf; + dtf = SQRT(dtf); + dtf = domain.arealg(indx) / dtf; - MinFinder tmp(dtf, i); - if(domain.vdov(indx) != Real_t(0.)) - { - minf += tmp; - } - }, - result); + MinFinder tmp(dtf, i); + if(domain.vdov(indx) != Real_t(0.)) + { + minf += tmp; + } + }, + result); dtcourant_tmp = result.val; @@ -2076,8 +2055,8 @@ CalcCourantConstraintForElems(Domain& domain, Index_t length, Index_t* regElemli } static inline void -CalcHydroConstraintForElems(Domain& domain, Index_t length, Index_t* regElemlist, - Real_t dvovmax, Real_t& dthydro) +CalcHydroConstraintForElems(Domain& domain, Index_t length, Index_t r, Real_t dvovmax, + Real_t& dthydro) { typedef Kokkos::View view_real_t; @@ -2085,23 +2064,23 @@ CalcHydroConstraintForElems(Domain& domain, Index_t length, Index_t* regElemlist Index_t hydro_elem = -1; MinFinder result; - Kokkos::parallel_reduce(length, - KOKKOS_LAMBDA(const int i, MinFinder& minf) { - Index_t indx = regElemlist[i]; + Kokkos::parallel_reduce( + length, + KOKKOS_LAMBDA(const int i, MinFinder& minf) { + Index_t indx = domain.regElemlist(r, i); - if(domain.vdov(indx) != Real_t(0.)) - { - Real_t dtdvov = dvovmax / (FABS(domain.vdov(indx)) + - Real_t(1.e-20)); + if(domain.vdov(indx) != Real_t(0.)) + { + Real_t dtdvov = dvovmax / (FABS(domain.vdov(indx)) + Real_t(1.e-20)); - MinFinder tmp(dtdvov, i); - if(domain.vdov(indx) != Real_t(0.)) - { - minf += tmp; - } - } - }, - result); + MinFinder tmp(dtdvov, i); + if(domain.vdov(indx) != Real_t(0.)) + { + minf += tmp; + } + } + }, + result); if(result.val > dthydro) { @@ -2124,12 +2103,11 @@ CalcTimeConstraintsForElems(Domain& domain) for(Index_t r = 0; r < domain.numReg(); ++r) { - CalcCourantConstraintForElems(domain, domain.regElemSize(r), - domain.regElemlist(r), domain.qqc(), + CalcCourantConstraintForElems(domain, domain.regElemSize(r), r, domain.qqc(), domain.dtcourant()); - CalcHydroConstraintForElems(domain, domain.regElemSize(r), domain.regElemlist(r), - domain.dvovmax(), domain.dthydro()); + CalcHydroConstraintForElems(domain, domain.regElemSize(r), r, domain.dvovmax(), + domain.dthydro()); } } @@ -2213,9 +2191,6 @@ main(int argc, char* argv[]) { printf("Running problem size %d^3 per domain until completion\n", opts.nx); printf("Num processors: %d\n", numRanks); -#if _OPENMP - printf("Num threads: %d\n", omp_get_max_threads()); -#endif printf("Total number of elements: %lld\n\n", (long long int) (numRanks * opts.nx * opts.nx * opts.nx)); printf("To run other sizes, use -s .\n"); @@ -2254,16 +2229,10 @@ main(int argc, char* argv[]) timeval start; gettimeofday(&start, NULL); #endif - Kokkos::Tools::pushRegion("Time-Loop"); while((locDom.time() < locDom.stoptime()) && (locDom.cycle() < opts.its)) { - Kokkos::Tools::pushRegion("TimeIncrement"); TimeIncrement(locDom); - Kokkos::Tools::popRegion(); - - Kokkos::Tools::pushRegion("LagrangeLeapFrog"); LagrangeLeapFrog(locDom); - Kokkos::Tools::popRegion(); if((opts.showProg != 0) && (opts.quiet == 0) && (myRank == 0)) { @@ -2271,7 +2240,6 @@ main(int argc, char* argv[]) double(locDom.time()), double(locDom.deltatime())); } } - Kokkos::Tools::popRegion(); double elapsed_time; #if USE_MPI @@ -2300,7 +2268,7 @@ main(int argc, char* argv[]) VerifyAndWriteFinalOutput(elapsed_timeG, locDom, opts.nx, numRanks); } - Release(&buffer); + buffer = Kokkos::View(); } Kokkos::finalize(); #if USE_MPI diff --git a/projects/rocprofiler-systems/examples/lulesh/lulesh.h b/projects/rocprofiler-systems/examples/lulesh/lulesh.h index 6c616b7700..e48e8edd3f 100644 --- a/projects/rocprofiler-systems/examples/lulesh/lulesh.h +++ b/projects/rocprofiler-systems/examples/lulesh/lulesh.h @@ -1,6 +1,6 @@ #if !defined(USE_MPI) -# error "You should specify USE_MPI=0 or USE_MPI=1 on the compile line" +#error "You should specify USE_MPI=0 or USE_MPI=1 on the compile line" #endif // OpenMP will be compiled in if this flag is set to 1 AND the compiler beging @@ -8,7 +8,7 @@ #define USE_OMP 1 #if USE_MPI -# include +#include /* define one of these three symbols: @@ -18,11 +18,12 @@ SEDOV_SYNC_POS_VEL_LATE */ -# define SEDOV_SYNC_POS_VEL_EARLY 1 +#define SEDOV_SYNC_POS_VEL_EARLY 1 #endif #include #include +#include #include #include @@ -34,67 +35,27 @@ #define MAX(a, b) (((a) > (b)) ? (a) : (b)) // Precision specification -typedef float real4; -typedef double real8; -typedef long double real10; // 10 bytes on x86 +typedef float real4; +typedef double real8; +typedef long double real10; // 10 bytes on x86 -typedef int Index_t; // array subscript and loop index -typedef real8 Real_t; // floating point representation -typedef int Int_t; // integer representation +typedef int Index_t; // array subscript and loop index +typedef real8 Real_t; // floating point representation +typedef int Int_t; // integer representation -enum -{ - VolumeError = -1, - QStopError = -2 -}; +enum { VolumeError = -1, QStopError = -2 }; -inline real4 -SQRT(real4 arg) -{ - return sqrtf(arg); -} -inline real8 -SQRT(real8 arg) -{ - return sqrt(arg); -} -inline real10 -SQRT(real10 arg) -{ - return sqrtl(arg); -} +KOKKOS_INLINE_FUNCTION real4 SQRT(real4 arg) { return sqrtf(arg); } +KOKKOS_INLINE_FUNCTION real8 SQRT(real8 arg) { return sqrt(arg); } +KOKKOS_INLINE_FUNCTION real10 SQRT(real10 arg) { return sqrtl(arg); } -inline real4 -CBRT(real4 arg) -{ - return cbrtf(arg); -} -inline real8 -CBRT(real8 arg) -{ - return cbrt(arg); -} -inline real10 -CBRT(real10 arg) -{ - return cbrtl(arg); -} +KOKKOS_INLINE_FUNCTION real4 CBRT(real4 arg) { return cbrtf(arg); } +KOKKOS_INLINE_FUNCTION real8 CBRT(real8 arg) { return cbrt(arg); } +KOKKOS_INLINE_FUNCTION real10 CBRT(real10 arg) { return cbrtl(arg); } -inline real4 -FABS(real4 arg) -{ - return fabsf(arg); -} -inline real8 -FABS(real8 arg) -{ - return fabs(arg); -} -inline real10 -FABS(real10 arg) -{ - return fabsl(arg); -} +KOKKOS_INLINE_FUNCTION real4 FABS(real4 arg) { return fabsf(arg); } +KOKKOS_INLINE_FUNCTION real8 FABS(real8 arg) { return fabs(arg); } +KOKKOS_INLINE_FUNCTION real10 FABS(real10 arg) { return fabsl(arg); } // Stuff needed for boundary conditions // 2 BCs on each of 6 hexahedral faces (12 bits) @@ -139,8 +100,8 @@ FABS(real10 arg) // Assume Real_t is an "integral power of 2" bytes wide #define CACHE_COHERENCE_PAD_REAL (128 / sizeof(Real_t)) -#define CACHE_ALIGN_REAL(n) \ - (((n) + (CACHE_COHERENCE_PAD_REAL - 1)) & ~(CACHE_COHERENCE_PAD_REAL - 1)) +#define CACHE_ALIGN_REAL(n) \ + (((n) + (CACHE_COHERENCE_PAD_REAL - 1)) & ~(CACHE_COHERENCE_PAD_REAL - 1)) ////////////////////////////////////////////////////// // Primary data structure @@ -165,550 +126,598 @@ FABS(real10 arg) * "Real_t &z(Index_t idx) { return m_coord[idx].z ; }" */ -class Domain -{ +class Domain { + public: - // Constructor - Domain(Int_t numRanks, Index_t colLoc, Index_t rowLoc, Index_t planeLoc, Index_t nx, - Int_t tp, Int_t nr, Int_t balance, Int_t cost); + // Constructor + Domain(Int_t numRanks, Index_t colLoc, Index_t rowLoc, Index_t planeLoc, + Index_t nx, Int_t tp, Int_t nr, Int_t balance, Int_t cost); - // Destructor - ~Domain(); + // Destructor + KOKKOS_FUNCTION ~Domain(); - // - // ALLOCATION - // + // + // ALLOCATION + // - void AllocateNodePersistent(Int_t numNode) // Node-centered - { - m_x.resize(numNode); // coordinates - m_y.resize(numNode); - m_z.resize(numNode); + void AllocateNodePersistent(Int_t numNode) // Node-centered + { + Kokkos::resize(m_x,numNode); // coordinates + Kokkos::resize(m_y,numNode); + Kokkos::resize(m_z,numNode); - m_xd.resize(numNode); // velocities - m_yd.resize(numNode); - m_zd.resize(numNode); + Kokkos::resize(m_xd,numNode); // velocities + Kokkos::resize(m_yd,numNode); + Kokkos::resize(m_zd,numNode); - m_xdd.resize(numNode); // accelerations - m_ydd.resize(numNode); - m_zdd.resize(numNode); + Kokkos::resize(m_xdd,numNode); // accelerations + Kokkos::resize(m_ydd,numNode); + Kokkos::resize(m_zdd,numNode); - m_fx.resize(numNode); // forces - m_fy.resize(numNode); - m_fz.resize(numNode); + Kokkos::resize(m_fx,numNode); // forces + Kokkos::resize(m_fy,numNode); + Kokkos::resize(m_fz,numNode); - m_nodalMass.resize(numNode); // mass + Kokkos::resize(m_nodalMass,numNode); // mass - m_c_x = m_x.d_view; - m_c_y = m_y.d_view; - m_c_z = m_z.d_view; - m_c_xd = m_xd.d_view; - m_c_yd = m_yd.d_view; - m_c_zd = m_zd.d_view; - } + m_c_x = m_x; + m_c_y = m_y; + m_c_z = m_z; + m_c_xd = m_xd; + m_c_yd = m_yd; + m_c_zd = m_zd; + } - void AllocateElemPersistent(Int_t numElem) // Elem-centered - { - m_nodelist.resize(8 * numElem); - - // elem connectivities through face - m_lxim.resize(numElem); - m_lxip.resize(numElem); - m_letam.resize(numElem); - m_letap.resize(numElem); - m_lzetam.resize(numElem); - m_lzetap.resize(numElem); - - m_elemBC.resize(numElem); - - m_e.resize(numElem); - m_p.resize(numElem); - - m_q.resize(numElem); - m_ql.resize(numElem); - m_qq.resize(numElem); - - m_v.resize(numElem); - - m_volo.resize(numElem); - m_delv.resize(numElem); - m_vdov.resize(numElem); - - m_arealg.resize(numElem); - - m_ss.resize(numElem); - - m_elemMass.resize(numElem); - - m_vnew.resize(numElem); - - m_c_e = m_e.d_view; - m_c_p = m_p.d_view; - m_c_q = m_q.d_view; - m_c_ql = m_ql.d_view; - m_c_qq = m_qq.d_view; - m_c_delv = m_delv.d_view; - } - - void AllocateGradients(Int_t numElem, Int_t allElem) - { - // Position gradients - m_delx_xi.resize(numElem); - m_delx_eta.resize(numElem); - m_delx_zeta.resize(numElem); - - // Velocity gradients - m_delv_xi.resize(allElem); - m_delv_eta.resize(allElem); - m_delv_zeta.resize(allElem); - } - - void DeallocateGradients() - { - m_delx_zeta.clear(); - m_delx_eta.clear(); - m_delx_xi.clear(); - - m_delv_zeta.clear(); - m_delv_eta.clear(); - m_delv_xi.clear(); - } - - void AllocateStrains(Int_t numElem) - { - m_dxx.resize(numElem); - m_dyy.resize(numElem); - m_dzz.resize(numElem); - } - - void DeallocateStrains() - { - m_dzz.clear(); - m_dyy.clear(); - m_dxx.clear(); - } - - // - // ACCESSORS - // - - // Node-centered - - // Nodal coordinates - KOKKOS_INLINE_FUNCTION Real_t& x(const Index_t idx) const { return m_x[idx]; } - KOKKOS_INLINE_FUNCTION Real_t& y(const Index_t idx) const { return m_y[idx]; } - KOKKOS_INLINE_FUNCTION Real_t& z(const Index_t idx) const { return m_z[idx]; } - KOKKOS_INLINE_FUNCTION Real_t c_x(const Index_t idx) const { return m_c_x[idx]; } - KOKKOS_INLINE_FUNCTION Real_t c_y(const Index_t idx) const { return m_c_y[idx]; } - KOKKOS_INLINE_FUNCTION Real_t c_z(const Index_t idx) const { return m_c_z[idx]; } - - // Nodal velocities - KOKKOS_INLINE_FUNCTION Real_t& xd(const Index_t idx) const { return m_xd[idx]; } - KOKKOS_INLINE_FUNCTION Real_t& yd(const Index_t idx) const { return m_yd[idx]; } - KOKKOS_INLINE_FUNCTION Real_t& zd(const Index_t idx) const { return m_zd[idx]; } - KOKKOS_INLINE_FUNCTION Real_t c_xd(const Index_t idx) const { return m_c_xd[idx]; } - KOKKOS_INLINE_FUNCTION Real_t c_yd(const Index_t idx) const { return m_c_yd[idx]; } - KOKKOS_INLINE_FUNCTION Real_t c_zd(const Index_t idx) const { return m_c_zd[idx]; } - - // Nodal accelerations - KOKKOS_INLINE_FUNCTION Real_t& xdd(const Index_t idx) const { return m_xdd[idx]; } - KOKKOS_INLINE_FUNCTION Real_t& ydd(const Index_t idx) const { return m_ydd[idx]; } - KOKKOS_INLINE_FUNCTION Real_t& zdd(const Index_t idx) const { return m_zdd[idx]; } - - // Nodal forces - KOKKOS_INLINE_FUNCTION Real_t& fx(const Index_t idx) const { return m_fx[idx]; } - KOKKOS_INLINE_FUNCTION Real_t& fy(const Index_t idx) const { return m_fy[idx]; } - KOKKOS_INLINE_FUNCTION Real_t& fz(const Index_t idx) const { return m_fz[idx]; } - - // Nodal mass - KOKKOS_INLINE_FUNCTION Real_t& nodalMass(const Index_t idx) const - { - return m_nodalMass[idx]; - } - - // Nodes on symmertry planes - Index_t symmX(const Index_t idx) const { return m_symmX[idx]; } - Index_t symmY(const Index_t idx) const { return m_symmY[idx]; } - Index_t symmZ(const Index_t idx) const { return m_symmZ[idx]; } - bool symmXempty() { return m_symmX.empty(); } - bool symmYempty() { return m_symmY.empty(); } - bool symmZempty() { return m_symmZ.empty(); } - - // - // Element-centered - // - Index_t& regElemSize(Index_t idx) { return m_regElemSize[idx]; } - Index_t& regNumList(Index_t idx) { return m_regNumList[idx]; } - Index_t* regNumList() { return &m_regNumList[0]; } - Index_t* regElemlist(Int_t r) { return m_regElemlist[r]; } - Index_t& regElemlist(const Int_t r, Index_t idx) const - { - return m_regElemlist[r][idx]; - } - - Index_t* nodelist(Index_t idx) const { return &m_nodelist[Index_t(8) * idx]; } + void AllocateElemPersistent(Int_t numElem) // Elem-centered + { + Kokkos::resize(m_nodelist,numElem); // elem connectivities through face - Index_t& lxim(const Index_t idx) const { return m_lxim[idx]; } - Index_t& lxip(const Index_t idx) const { return m_lxip[idx]; } - Index_t& letam(const Index_t idx) const { return m_letam[idx]; } - Index_t& letap(const Index_t idx) const { return m_letap[idx]; } - Index_t& lzetam(const Index_t idx) const { return m_lzetam[idx]; } - Index_t& lzetap(const Index_t idx) const { return m_lzetap[idx]; } + Kokkos::resize(m_lxim,numElem); + Kokkos::resize(m_lxip,numElem); + Kokkos::resize(m_letam,numElem); + Kokkos::resize(m_letap,numElem); + Kokkos::resize(m_lzetam,numElem); + Kokkos::resize(m_lzetap,numElem); - // elem face symm/free-surface flag - Int_t& elemBC(const Index_t idx) const { return m_elemBC[idx]; } + Kokkos::resize(m_elemBC,numElem); - // Principal strains - temporary - KOKKOS_INLINE_FUNCTION Real_t& dxx(const Index_t idx) const { return m_dxx[idx]; } - KOKKOS_INLINE_FUNCTION Real_t& dyy(const Index_t idx) const { return m_dyy[idx]; } - KOKKOS_INLINE_FUNCTION Real_t& dzz(const Index_t idx) const { return m_dzz[idx]; } + Kokkos::resize(m_e,numElem); + Kokkos::resize(m_p,numElem); - // New relative volume - temporary - KOKKOS_INLINE_FUNCTION Real_t& vnew(const Index_t idx) const { return m_vnew[idx]; } + Kokkos::resize(m_q,numElem); + Kokkos::resize(m_ql,numElem); + Kokkos::resize(m_qq,numElem); - // Velocity gradient - temporary - KOKKOS_INLINE_FUNCTION Real_t& delv_xi(const Index_t idx) const - { - return m_delv_xi[idx]; - } - KOKKOS_INLINE_FUNCTION Real_t& delv_eta(const Index_t idx) const - { - return m_delv_eta[idx]; - } - KOKKOS_INLINE_FUNCTION Real_t& delv_zeta(const Index_t idx) const - { - return m_delv_zeta[idx]; - } + Kokkos::resize(m_v,numElem); - // Position gradient - temporary - KOKKOS_INLINE_FUNCTION Real_t& delx_xi(const Index_t idx) const - { - return m_delx_xi[idx]; - } - KOKKOS_INLINE_FUNCTION Real_t& delx_eta(const Index_t idx) const - { - return m_delx_eta[idx]; - } - KOKKOS_INLINE_FUNCTION Real_t& delx_zeta(const Index_t idx) const - { - return m_delx_zeta[idx]; - } - // Energy - KOKKOS_INLINE_FUNCTION Real_t& e(const Index_t idx) const { return m_e[idx]; } - KOKKOS_INLINE_FUNCTION Real_t c_e(const Index_t idx) const { return m_c_e[idx]; } + Kokkos::resize(m_volo,numElem); + Kokkos::resize(m_delv,numElem); + Kokkos::resize(m_vdov,numElem); - // Pressure - KOKKOS_INLINE_FUNCTION Real_t& p(const Index_t idx) const { return m_p[idx]; } - KOKKOS_INLINE_FUNCTION Real_t c_p(const Index_t idx) const { return m_c_p[idx]; } + Kokkos::resize(m_arealg,numElem); - // Artificial viscosity - KOKKOS_INLINE_FUNCTION Real_t& q(const Index_t idx) const { return m_q[idx]; } - KOKKOS_INLINE_FUNCTION Real_t c_q(const Index_t idx) const { return m_c_q[idx]; } + Kokkos::resize(m_ss,numElem); - // Linear term for q - KOKKOS_INLINE_FUNCTION Real_t& ql(const Index_t idx) const { return m_ql[idx]; } - KOKKOS_INLINE_FUNCTION Real_t c_ql(const Index_t idx) const { return m_c_ql[idx]; } - // Quadratic term for q - KOKKOS_INLINE_FUNCTION Real_t& qq(const Index_t idx) const { return m_qq[idx]; } - KOKKOS_INLINE_FUNCTION Real_t c_qq(const Index_t idx) const { return m_c_qq[idx]; } + Kokkos::resize(m_elemMass,numElem); - // Relative volume - KOKKOS_INLINE_FUNCTION Real_t& v(const Index_t idx) const { return m_v[idx]; } - KOKKOS_INLINE_FUNCTION Real_t& delv(const Index_t idx) const { return m_delv[idx]; } - KOKKOS_INLINE_FUNCTION Real_t c_delv(const Index_t idx) const - { - return m_c_delv[idx]; - } + Kokkos::resize(m_vnew,numElem); - // Reference volume - KOKKOS_INLINE_FUNCTION Real_t& volo(Index_t idx) const { return m_volo[idx]; } + m_c_e = m_e; + m_c_p = m_p; + m_c_q = m_q; + m_c_ql = m_ql; + m_c_qq = m_qq; + m_c_delv = m_delv; + } - // volume derivative over volume - KOKKOS_INLINE_FUNCTION Real_t& vdov(Index_t idx) const { return m_vdov[idx]; } + void AllocateGradients(Int_t numElem, Int_t allElem) { + // Position gradients + Kokkos::resize(m_delx_xi,numElem); + Kokkos::resize(m_delx_eta,numElem); + Kokkos::resize(m_delx_zeta,numElem); - // Element characteristic length - KOKKOS_INLINE_FUNCTION Real_t& arealg(Index_t idx) const { return m_arealg[idx]; } + // Velocity gradients + Kokkos::resize(m_delv_xi,allElem); + Kokkos::resize(m_delv_eta,allElem); + Kokkos::resize(m_delv_zeta,allElem); + } - // Sound speed - KOKKOS_INLINE_FUNCTION Real_t& ss(const Index_t idx) const { return m_ss[idx]; } + void DeallocateGradients() { + m_delx_zeta = Kokkos::View(); + m_delx_eta = Kokkos::View(); + m_delx_xi = Kokkos::View(); - // Element mass - KOKKOS_INLINE_FUNCTION Real_t& elemMass(const Index_t idx) const - { - return m_elemMass[idx]; - } + m_delv_zeta = Kokkos::View(); + m_delv_eta = Kokkos::View(); + m_delv_xi = Kokkos::View(); + } - KOKKOS_INLINE_FUNCTION Index_t nodeElemCount(Index_t idx) const - { - return m_nodeElemStart[idx + 1] - m_nodeElemStart[idx]; - } + void AllocateStrains(Int_t numElem) { + Kokkos::resize(m_dxx,numElem); + Kokkos::resize(m_dyy,numElem); + Kokkos::resize(m_dzz,numElem); + } - KOKKOS_INLINE_FUNCTION Index_t* nodeElemCornerList(Index_t idx) const - { - return &m_nodeElemCornerList[m_nodeElemStart[idx]]; - } + void DeallocateStrains() { + m_dzz = Kokkos::View(); + m_dyy = Kokkos::View(); + m_dxx = Kokkos::View(); + } - // Parameters + // + // ACCESSORS + // + KOKKOS_INLINE_FUNCTION + const Kokkos::View& e_view() const { return m_e; } - // Cutoffs - KOKKOS_INLINE_FUNCTION Real_t u_cut() const { return m_u_cut; } - KOKKOS_INLINE_FUNCTION Real_t e_cut() const { return m_e_cut; } - KOKKOS_INLINE_FUNCTION Real_t p_cut() const { return m_p_cut; } - KOKKOS_INLINE_FUNCTION Real_t q_cut() const { return m_q_cut; } - KOKKOS_INLINE_FUNCTION Real_t v_cut() const { return m_v_cut; } + // Node-centered - // Other constants (usually are settable via input file in real codes) - KOKKOS_INLINE_FUNCTION Real_t hgcoef() const { return m_hgcoef; } - KOKKOS_INLINE_FUNCTION Real_t qstop() const { return m_qstop; } - KOKKOS_INLINE_FUNCTION Real_t monoq_max_slope() const { return m_monoq_max_slope; } - KOKKOS_INLINE_FUNCTION Real_t monoq_limiter_mult() const - { - return m_monoq_limiter_mult; - } - KOKKOS_INLINE_FUNCTION Real_t ss4o3() const { return m_ss4o3; } - KOKKOS_INLINE_FUNCTION Real_t qlc_monoq() const { return m_qlc_monoq; } - KOKKOS_INLINE_FUNCTION Real_t qqc_monoq() const { return m_qqc_monoq; } - KOKKOS_INLINE_FUNCTION Real_t qqc() const { return m_qqc; } + // Nodal coordinates + KOKKOS_INLINE_FUNCTION Real_t &x(const Index_t idx) const { return m_x[idx]; } + KOKKOS_INLINE_FUNCTION Real_t &y(const Index_t idx) const { return m_y[idx]; } + KOKKOS_INLINE_FUNCTION Real_t &z(const Index_t idx) const { return m_z[idx]; } + KOKKOS_INLINE_FUNCTION Real_t c_x(const Index_t idx) const { + return m_c_x[idx]; + } + KOKKOS_INLINE_FUNCTION Real_t c_y(const Index_t idx) const { + return m_c_y[idx]; + } + KOKKOS_INLINE_FUNCTION Real_t c_z(const Index_t idx) const { + return m_c_z[idx]; + } - KOKKOS_INLINE_FUNCTION Real_t eosvmax() const { return m_eosvmax; } - KOKKOS_INLINE_FUNCTION Real_t eosvmin() const { return m_eosvmin; } - KOKKOS_INLINE_FUNCTION Real_t pmin() const { return m_pmin; } - KOKKOS_INLINE_FUNCTION Real_t emin() const { return m_emin; } - KOKKOS_INLINE_FUNCTION Real_t dvovmax() const { return m_dvovmax; } - KOKKOS_INLINE_FUNCTION Real_t refdens() const { return m_refdens; } + // Nodal velocities + KOKKOS_INLINE_FUNCTION Real_t &xd(const Index_t idx) const { + return m_xd[idx]; + } + KOKKOS_INLINE_FUNCTION Real_t &yd(const Index_t idx) const { + return m_yd[idx]; + } + KOKKOS_INLINE_FUNCTION Real_t &zd(const Index_t idx) const { + return m_zd[idx]; + } + KOKKOS_INLINE_FUNCTION Real_t c_xd(const Index_t idx) const { + return m_c_xd[idx]; + } + KOKKOS_INLINE_FUNCTION Real_t c_yd(const Index_t idx) const { + return m_c_yd[idx]; + } + KOKKOS_INLINE_FUNCTION Real_t c_zd(const Index_t idx) const { + return m_c_zd[idx]; + } - // Timestep controls, etc... - Real_t& time() { return m_time; } - Real_t& deltatime() { return m_deltatime; } - Real_t& deltatimemultlb() { return m_deltatimemultlb; } - Real_t& deltatimemultub() { return m_deltatimemultub; } - Real_t& stoptime() { return m_stoptime; } - Real_t& dtcourant() { return m_dtcourant; } - Real_t& dthydro() { return m_dthydro; } - Real_t& dtmax() { return m_dtmax; } - Real_t& dtfixed() { return m_dtfixed; } + // Nodal accelerations + KOKKOS_INLINE_FUNCTION Real_t &xdd(const Index_t idx) const { + return m_xdd[idx]; + } + KOKKOS_INLINE_FUNCTION Real_t &ydd(const Index_t idx) const { + return m_ydd[idx]; + } + KOKKOS_INLINE_FUNCTION Real_t &zdd(const Index_t idx) const { + return m_zdd[idx]; + } - Int_t& cycle() { return m_cycle; } - Index_t& numRanks() { return m_numRanks; } + // Nodal forces + KOKKOS_INLINE_FUNCTION Real_t &fx(const Index_t idx) const { + return m_fx[idx]; + } + KOKKOS_INLINE_FUNCTION Real_t &fy(const Index_t idx) const { + return m_fy[idx]; + } + KOKKOS_INLINE_FUNCTION Real_t &fz(const Index_t idx) const { + return m_fz[idx]; + } - Index_t& colLoc() { return m_colLoc; } - Index_t& rowLoc() { return m_rowLoc; } - Index_t& planeLoc() { return m_planeLoc; } - Index_t& tp() { return m_tp; } + // Nodal mass + KOKKOS_INLINE_FUNCTION Real_t &nodalMass(const Index_t idx) const { + return m_nodalMass[idx]; + } - Index_t& sizeX() { return m_sizeX; } - Index_t& sizeY() { return m_sizeY; } - Index_t& sizeZ() { return m_sizeZ; } - Index_t& numReg() { return m_numReg; } - Int_t& cost() { return m_cost; } - Index_t& numElem() { return m_numElem; } - Index_t& numNode() { return m_numNode; } + // Nodes on symmertry planes + KOKKOS_INLINE_FUNCTION Index_t symmX(const Index_t idx) const { return m_symmX[idx]; } + KOKKOS_INLINE_FUNCTION Index_t symmY(const Index_t idx) const { return m_symmY[idx]; } + KOKKOS_INLINE_FUNCTION Index_t symmZ(const Index_t idx) const { return m_symmZ[idx]; } + KOKKOS_INLINE_FUNCTION bool symmXempty() { return m_symmX.data()==nullptr; } + KOKKOS_INLINE_FUNCTION bool symmYempty() { return m_symmY.data()==nullptr; } + KOKKOS_INLINE_FUNCTION bool symmZempty() { return m_symmZ.data()==nullptr; } - Index_t& maxPlaneSize() { return m_maxPlaneSize; } - Index_t& maxEdgeSize() { return m_maxEdgeSize; } + // + // Element-centered + // + Index_t ®ElemSize(Index_t idx) { return m_regElemSize[idx]; } + Index_t ®NumList(Index_t idx) { return m_regNumList[idx]; } + Index_t *regNumList() { return &m_regNumList[0]; } + Index_t *regElemlist(Int_t r) const { return &m_regElemlist.entries(m_regElemlist.row_map(r)); } + KOKKOS_INLINE_FUNCTION Index_t regElemlist(const Int_t r, Index_t idx) const { + return m_regElemlist.entries(m_regElemlist.row_map(r)+idx); + } - // - // MPI-Related additional data - // + KOKKOS_INLINE_FUNCTION Index_t &nodelist(Index_t i, Index_t j) const { return m_nodelist(i,j); } + + + // elem connectivities through face + KOKKOS_INLINE_FUNCTION Index_t &lxim(const Index_t idx) const { return m_lxim[idx]; } + KOKKOS_INLINE_FUNCTION Index_t &lxip(const Index_t idx) const { return m_lxip[idx]; } + KOKKOS_INLINE_FUNCTION Index_t &letam(const Index_t idx) const { return m_letam[idx]; } + KOKKOS_INLINE_FUNCTION Index_t &letap(const Index_t idx) const { return m_letap[idx]; } + KOKKOS_INLINE_FUNCTION Index_t &lzetam(const Index_t idx) const { return m_lzetam[idx]; } + KOKKOS_INLINE_FUNCTION Index_t &lzetap(const Index_t idx) const { return m_lzetap[idx]; } + + // elem face symm/free-surface flag + KOKKOS_INLINE_FUNCTION Int_t &elemBC(const Index_t idx) const { return m_elemBC[idx]; } + + // Principal strains - temporary + KOKKOS_INLINE_FUNCTION Real_t &dxx(const Index_t idx) const { + return m_dxx[idx]; + } + KOKKOS_INLINE_FUNCTION Real_t &dyy(const Index_t idx) const { + return m_dyy[idx]; + } + KOKKOS_INLINE_FUNCTION Real_t &dzz(const Index_t idx) const { + return m_dzz[idx]; + } + + // New relative volume - temporary + KOKKOS_INLINE_FUNCTION Real_t &vnew(const Index_t idx) const { + return m_vnew[idx]; + } + + // Velocity gradient - temporary + KOKKOS_INLINE_FUNCTION Real_t &delv_xi(const Index_t idx) const { + return m_delv_xi[idx]; + } + KOKKOS_INLINE_FUNCTION Real_t &delv_eta(const Index_t idx) const { + return m_delv_eta[idx]; + } + KOKKOS_INLINE_FUNCTION Real_t &delv_zeta(const Index_t idx) const { + return m_delv_zeta[idx]; + } + + // Position gradient - temporary + KOKKOS_INLINE_FUNCTION Real_t &delx_xi(const Index_t idx) const { + return m_delx_xi[idx]; + } + KOKKOS_INLINE_FUNCTION Real_t &delx_eta(const Index_t idx) const { + return m_delx_eta[idx]; + } + KOKKOS_INLINE_FUNCTION Real_t &delx_zeta(const Index_t idx) const { + return m_delx_zeta[idx]; + } + // Energy + KOKKOS_INLINE_FUNCTION Real_t &e(const Index_t idx) const { return m_e[idx]; } + KOKKOS_INLINE_FUNCTION Real_t c_e(const Index_t idx) const { + return m_c_e[idx]; + } + + // Pressure + KOKKOS_INLINE_FUNCTION Real_t &p(const Index_t idx) const { return m_p[idx]; } + KOKKOS_INLINE_FUNCTION Real_t c_p(const Index_t idx) const { + return m_c_p[idx]; + } + + // Artificial viscosity + KOKKOS_INLINE_FUNCTION Real_t &q(const Index_t idx) const { return m_q[idx]; } + KOKKOS_INLINE_FUNCTION Real_t c_q(const Index_t idx) const { + return m_c_q[idx]; + } + + // Linear term for q + KOKKOS_INLINE_FUNCTION Real_t &ql(const Index_t idx) const { + return m_ql[idx]; + } + KOKKOS_INLINE_FUNCTION Real_t c_ql(const Index_t idx) const { + return m_c_ql[idx]; + } + // Quadratic term for q + KOKKOS_INLINE_FUNCTION Real_t &qq(const Index_t idx) const { + return m_qq[idx]; + } + KOKKOS_INLINE_FUNCTION Real_t c_qq(const Index_t idx) const { + return m_c_qq[idx]; + } + + // Relative volume + KOKKOS_INLINE_FUNCTION Real_t &v(const Index_t idx) const { return m_v[idx]; } + KOKKOS_INLINE_FUNCTION Real_t &delv(const Index_t idx) const { + return m_delv[idx]; + } + KOKKOS_INLINE_FUNCTION Real_t c_delv(const Index_t idx) const { + return m_c_delv[idx]; + } + + // Reference volume + KOKKOS_INLINE_FUNCTION Real_t &volo(Index_t idx) const { return m_volo[idx]; } + + // volume derivative over volume + KOKKOS_INLINE_FUNCTION Real_t &vdov(Index_t idx) const { return m_vdov[idx]; } + + // Element characteristic length + KOKKOS_INLINE_FUNCTION Real_t &arealg(Index_t idx) const { + return m_arealg[idx]; + } + + // Sound speed + KOKKOS_INLINE_FUNCTION Real_t &ss(const Index_t idx) const { + return m_ss[idx]; + } + + // Element mass + KOKKOS_INLINE_FUNCTION Real_t &elemMass(const Index_t idx) const { + return m_elemMass[idx]; + } + + KOKKOS_INLINE_FUNCTION Index_t nodeElemCount(Index_t idx) const { + return m_nodeElemStart[idx + 1] - m_nodeElemStart[idx]; + } + + KOKKOS_INLINE_FUNCTION Index_t *nodeElemCornerList(Index_t idx) const { + return &m_nodeElemCornerList[m_nodeElemStart[idx]]; + } + + // Parameters + + // Cutoffs + KOKKOS_INLINE_FUNCTION Real_t u_cut() const { return m_u_cut; } + KOKKOS_INLINE_FUNCTION Real_t e_cut() const { return m_e_cut; } + KOKKOS_INLINE_FUNCTION Real_t p_cut() const { return m_p_cut; } + KOKKOS_INLINE_FUNCTION Real_t q_cut() const { return m_q_cut; } + KOKKOS_INLINE_FUNCTION Real_t v_cut() const { return m_v_cut; } + + // Other constants (usually are settable via input file in real codes) + KOKKOS_INLINE_FUNCTION Real_t hgcoef() const { return m_hgcoef; } + KOKKOS_INLINE_FUNCTION Real_t qstop() const { return m_qstop; } + KOKKOS_INLINE_FUNCTION Real_t monoq_max_slope() const { + return m_monoq_max_slope; + } + KOKKOS_INLINE_FUNCTION Real_t monoq_limiter_mult() const { + return m_monoq_limiter_mult; + } + KOKKOS_INLINE_FUNCTION Real_t ss4o3() const { return m_ss4o3; } + KOKKOS_INLINE_FUNCTION Real_t qlc_monoq() const { return m_qlc_monoq; } + KOKKOS_INLINE_FUNCTION Real_t qqc_monoq() const { return m_qqc_monoq; } + KOKKOS_INLINE_FUNCTION Real_t qqc() const { return m_qqc; } + + KOKKOS_INLINE_FUNCTION Real_t eosvmax() const { return m_eosvmax; } + KOKKOS_INLINE_FUNCTION Real_t eosvmin() const { return m_eosvmin; } + KOKKOS_INLINE_FUNCTION Real_t pmin() const { return m_pmin; } + KOKKOS_INLINE_FUNCTION Real_t emin() const { return m_emin; } + KOKKOS_INLINE_FUNCTION Real_t dvovmax() const { return m_dvovmax; } + KOKKOS_INLINE_FUNCTION Real_t refdens() const { return m_refdens; } + + // Timestep controls, etc... + Real_t &time() { return m_time; } + Real_t &deltatime() { return m_deltatime; } + Real_t &deltatimemultlb() { return m_deltatimemultlb; } + Real_t &deltatimemultub() { return m_deltatimemultub; } + Real_t &stoptime() { return m_stoptime; } + Real_t &dtcourant() { return m_dtcourant; } + Real_t &dthydro() { return m_dthydro; } + Real_t &dtmax() { return m_dtmax; } + Real_t &dtfixed() { return m_dtfixed; } + + Int_t &cycle() { return m_cycle; } + Index_t &numRanks() { return m_numRanks; } + + Index_t &colLoc() { return m_colLoc; } + Index_t &rowLoc() { return m_rowLoc; } + Index_t &planeLoc() { return m_planeLoc; } + Index_t &tp() { return m_tp; } + + Index_t &sizeX() { return m_sizeX; } + Index_t &sizeY() { return m_sizeY; } + Index_t &sizeZ() { return m_sizeZ; } + Index_t &numReg() { return m_numReg; } + Int_t &cost() { return m_cost; } + Index_t &numElem() { return m_numElem; } + Index_t &numNode() { return m_numNode; } + + Index_t &maxPlaneSize() { return m_maxPlaneSize; } + Index_t &maxEdgeSize() { return m_maxEdgeSize; } + +// +// MPI-Related additional data +// #if USE_MPI - // Communication Work space - Real_t* commDataSend; - Real_t* commDataRecv; + // Communication Work space + Real_t *commDataSend; + Real_t *commDataRecv; - // Maximum number of block neighbors - MPI_Request recvRequest[26]; // 6 faces + 12 edges + 8 corners - MPI_Request sendRequest[26]; // 6 faces + 12 edges + 8 corners + // Maximum number of block neighbors + MPI_Request recvRequest[26]; // 6 faces + 12 edges + 8 corners + MPI_Request sendRequest[26]; // 6 faces + 12 edges + 8 corners #endif private: - void BuildMesh(Int_t nx, Int_t edgeNodes, Int_t edgeElems); - void SetupThreadSupportStructures(); - void CreateRegionIndexSets(Int_t nreg, Int_t balance); - void SetupCommBuffers(Int_t edgeNodes); - void SetupSymmetryPlanes(Int_t edgeNodes); - void SetupElementConnectivities(Int_t edgeElems); - void SetupBoundaryConditions(Int_t edgeElems); + void BuildMesh(Int_t nx, Int_t edgeNodes, Int_t edgeElems); + void SetupThreadSupportStructures(); + void CreateRegionIndexSets(Int_t nreg, Int_t balance); + void SetupCommBuffers(Int_t edgeNodes); + void SetupSymmetryPlanes(Int_t edgeNodes); + void SetupElementConnectivities(Int_t edgeElems); + void SetupBoundaryConditions(Int_t edgeElems); - // - // IMPLEMENTATION - // + // + // IMPLEMENTATION + // - /* Node-centered */ - Kokkos::vector m_x; /* coordinates */ - Kokkos::vector m_y; - Kokkos::vector m_z; - Kokkos::View> - m_c_x; /* coordinates */ - Kokkos::View> - m_c_y; /* coordinates */ - Kokkos::View> - m_c_z; /* coordinates */ + /* Node-centered */ + Kokkos::View m_x; /* coordinates */ + Kokkos::View m_y; + Kokkos::View m_z; + Kokkos::View> + m_c_x; /* coordinates */ + Kokkos::View> + m_c_y; /* coordinates */ + Kokkos::View> + m_c_z; /* coordinates */ - Kokkos::vector m_xd; /* velocities */ - Kokkos::vector m_yd; - Kokkos::vector m_zd; - Kokkos::View> - m_c_xd; /* coordinates */ - Kokkos::View> - m_c_yd; /* coordinates */ - Kokkos::View> - m_c_zd; /* coordinates */ + Kokkos::View m_xd; /* velocities */ + Kokkos::View m_yd; + Kokkos::View m_zd; + Kokkos::View> + m_c_xd; /* coordinates */ + Kokkos::View> + m_c_yd; /* coordinates */ + Kokkos::View> + m_c_zd; /* coordinates */ - Kokkos::vector m_xdd; /* accelerations */ - Kokkos::vector m_ydd; - Kokkos::vector m_zdd; + Kokkos::View m_xdd; /* accelerations */ + Kokkos::View m_ydd; + Kokkos::View m_zdd; - Kokkos::vector m_fx; /* forces */ - Kokkos::vector m_fy; - Kokkos::vector m_fz; + Kokkos::View m_fx; /* forces */ + Kokkos::View m_fy; + Kokkos::View m_fz; - Kokkos::vector m_nodalMass; /* mass */ + Kokkos::View m_nodalMass; /* mass */ - Kokkos::vector m_symmX; /* symmetry plane nodesets */ - Kokkos::vector m_symmY; - Kokkos::vector m_symmZ; + Kokkos::View m_symmX; /* symmetry plane nodesets */ + Kokkos::View m_symmY; + Kokkos::View m_symmZ; - // Element-centered + // Element-centered - // Region information - Int_t m_numReg; - Int_t m_cost; // imbalance cost - Index_t* m_regElemSize; // Size of region sets - Index_t* m_regNumList; // Region number per domain element - Index_t** m_regElemlist; // region indexset + // Region information + Int_t m_numReg; + Int_t m_cost; // imbalance cost + Index_t *m_regElemSize; // Size of region sets + Index_t *m_regNumList; // Region number per domain element + //Index_t **m_regElemlist; // region indexset + using t_regElemlist = Kokkos::StaticCrsGraph,Index_t>; + t_regElemlist m_regElemlist; - Kokkos::vector m_nodelist; /* elemToNode connectivity */ + Kokkos::View m_nodelist; /* elemToNode connectivity */ - Kokkos::vector m_lxim; /* element connectivity across each face */ - Kokkos::vector m_lxip; - Kokkos::vector m_letam; - Kokkos::vector m_letap; - Kokkos::vector m_lzetam; - Kokkos::vector m_lzetap; + Kokkos::View m_lxim; /* element connectivity across each face */ + Kokkos::View m_lxip; + Kokkos::View m_letam; + Kokkos::View m_letap; + Kokkos::View m_lzetam; + Kokkos::View m_lzetap; - Kokkos::vector m_elemBC; /* symmetry/free-surface flags for each elem face */ + Kokkos::View + m_elemBC; /* symmetry/free-surface flags for each elem face */ - Kokkos::vector m_dxx; /* principal strains -- temporary */ - Kokkos::vector m_dyy; - Kokkos::vector m_dzz; + Kokkos::View m_dxx; /* principal strains -- temporary */ + Kokkos::View m_dyy; + Kokkos::View m_dzz; - Kokkos::vector m_delv_xi; /* velocity gradient -- temporary */ - Kokkos::vector m_delv_eta; - Kokkos::vector m_delv_zeta; + Kokkos::View m_delv_xi; /* velocity gradient -- temporary */ + Kokkos::View m_delv_eta; + Kokkos::View m_delv_zeta; - Kokkos::vector m_delx_xi; /* coordinate gradient -- temporary */ - Kokkos::vector m_delx_eta; - Kokkos::vector m_delx_zeta; + Kokkos::View m_delx_xi; /* coordinate gradient -- temporary */ + Kokkos::View m_delx_eta; + Kokkos::View m_delx_zeta; - Kokkos::vector m_e; /* energy */ + Kokkos::View m_e; /* energy */ - Kokkos::vector m_p; /* pressure */ - Kokkos::vector m_q; /* q */ - Kokkos::vector m_ql; /* linear term for q */ - Kokkos::vector m_qq; /* quadratic term for q */ + Kokkos::View m_p; /* pressure */ + Kokkos::View m_q; /* q */ + Kokkos::View m_ql; /* linear term for q */ + Kokkos::View m_qq; /* quadratic term for q */ - Kokkos::vector m_v; /* relative volume */ - Kokkos::vector m_volo; /* reference volume */ - Kokkos::vector m_vnew; /* new relative volume -- temporary */ - Kokkos::vector m_delv; /* m_vnew - m_v */ - Kokkos::vector m_vdov; /* volume derivative over volume */ + Kokkos::View m_v; /* relative volume */ + Kokkos::View m_volo; /* reference volume */ + Kokkos::View m_vnew; /* new relative volume -- temporary */ + Kokkos::View m_delv; /* m_vnew - m_v */ + Kokkos::View m_vdov; /* volume derivative over volume */ - Kokkos::View> - m_c_e; /* coordinates */ - Kokkos::View> - m_c_p; /* coordinates */ - Kokkos::View> - m_c_q; /* coordinates */ - Kokkos::View> - m_c_ql; /* coordinates */ - Kokkos::View> - m_c_qq; /* coordinates */ - Kokkos::View> - m_c_delv; /* coordinates */ + Kokkos::View> + m_c_e; /* coordinates */ + Kokkos::View> + m_c_p; /* coordinates */ + Kokkos::View> + m_c_q; /* coordinates */ + Kokkos::View> + m_c_ql; /* coordinates */ + Kokkos::View> + m_c_qq; /* coordinates */ + Kokkos::View> + m_c_delv; /* coordinates */ - Kokkos::vector m_arealg; /* characteristic length of an element */ + Kokkos::View m_arealg; /* characteristic length of an element */ - Kokkos::vector m_ss; /* "sound speed" */ + Kokkos::View m_ss; /* "sound speed" */ - Kokkos::vector m_elemMass; /* mass */ + Kokkos::View m_elemMass; /* mass */ - // Cutoffs (treat as constants) - const Real_t m_e_cut; // energy tolerance - const Real_t m_p_cut; // pressure tolerance - const Real_t m_q_cut; // q tolerance - const Real_t m_v_cut; // relative volume tolerance - const Real_t m_u_cut; // velocity tolerance + // Cutoffs (treat as constants) + const Real_t m_e_cut; // energy tolerance + const Real_t m_p_cut; // pressure tolerance + const Real_t m_q_cut; // q tolerance + const Real_t m_v_cut; // relative volume tolerance + const Real_t m_u_cut; // velocity tolerance - // Other constants (usually setable, but hardcoded in this proxy app) + // Other constants (usually setable, but hardcoded in this proxy app) - const Real_t m_hgcoef; // hourglass control - const Real_t m_ss4o3; - const Real_t m_qstop; // excessive q indicator - const Real_t m_monoq_max_slope; - const Real_t m_monoq_limiter_mult; - const Real_t m_qlc_monoq; // linear term coef for q - const Real_t m_qqc_monoq; // quadratic term coef for q - const Real_t m_qqc; - const Real_t m_eosvmax; - const Real_t m_eosvmin; - const Real_t m_pmin; // pressure floor - const Real_t m_emin; // energy floor - const Real_t m_dvovmax; // maximum allowable volume change - const Real_t m_refdens; // reference density + const Real_t m_hgcoef; // hourglass control + const Real_t m_ss4o3; + const Real_t m_qstop; // excessive q indicator + const Real_t m_monoq_max_slope; + const Real_t m_monoq_limiter_mult; + const Real_t m_qlc_monoq; // linear term coef for q + const Real_t m_qqc_monoq; // quadratic term coef for q + const Real_t m_qqc; + const Real_t m_eosvmax; + const Real_t m_eosvmin; + const Real_t m_pmin; // pressure floor + const Real_t m_emin; // energy floor + const Real_t m_dvovmax; // maximum allowable volume change + const Real_t m_refdens; // reference density - // Variables to keep track of timestep, simulation time, and cycle - Real_t m_dtcourant; // courant constraint - Real_t m_dthydro; // volume change constraint - Int_t m_cycle; // iteration count for simulation - Real_t m_dtfixed; // fixed time increment - Real_t m_time; // current time - Real_t m_deltatime; // variable time increment - Real_t m_deltatimemultlb; - Real_t m_deltatimemultub; - Real_t m_dtmax; // maximum allowable time increment - Real_t m_stoptime; // end time for simulation + // Variables to keep track of timestep, simulation time, and cycle + Real_t m_dtcourant; // courant constraint + Real_t m_dthydro; // volume change constraint + Int_t m_cycle; // iteration count for simulation + Real_t m_dtfixed; // fixed time increment + Real_t m_time; // current time + Real_t m_deltatime; // variable time increment + Real_t m_deltatimemultlb; + Real_t m_deltatimemultub; + Real_t m_dtmax; // maximum allowable time increment + Real_t m_stoptime; // end time for simulation - Int_t m_numRanks; + Int_t m_numRanks; - Index_t m_colLoc; - Index_t m_rowLoc; - Index_t m_planeLoc; - Index_t m_tp; + Index_t m_colLoc; + Index_t m_rowLoc; + Index_t m_planeLoc; + Index_t m_tp; - Index_t m_sizeX; - Index_t m_sizeY; - Index_t m_sizeZ; - Index_t m_numElem; - Index_t m_numNode; + Index_t m_sizeX; + Index_t m_sizeY; + Index_t m_sizeZ; + Index_t m_numElem; + Index_t m_numNode; - Index_t m_maxPlaneSize; - Index_t m_maxEdgeSize; + Index_t m_maxPlaneSize; + Index_t m_maxEdgeSize; - // OMP hack - Index_t* m_nodeElemStart; - Index_t* m_nodeElemCornerList; + // OMP hack + Kokkos::View m_nodeElemStart; + Kokkos::View m_nodeElemCornerList; - // Used in setup - Index_t m_rowMin, m_rowMax; - Index_t m_colMin, m_colMax; - Index_t m_planeMin, m_planeMax; + // Used in setup + Index_t m_rowMin, m_rowMax; + Index_t m_colMin, m_colMax; + Index_t m_planeMin, m_planeMax; }; -typedef Real_t& (Domain::*Domain_member)(Index_t) const; +typedef Real_t &(Domain::*Domain_member)(Index_t) const; -struct cmdLineOpts -{ - Int_t its; // -i - Int_t nx; // -s - Int_t numReg; // -r - Int_t numFiles; // -f - Int_t showProg; // -p - Int_t quiet; // -q - Int_t viz; // -v - Int_t cost; // -c - Int_t balance; // -b - Int_t do_atomic; // -a +struct cmdLineOpts { + Int_t its; // -i + Int_t nx; // -s + Int_t numReg; // -r + Int_t numFiles; // -f + Int_t showProg; // -p + Int_t quiet; // -q + Int_t viz; // -v + Int_t cost; // -c + Int_t balance; // -b + Int_t do_atomic; // -a }; // Function Prototypes @@ -719,33 +728,27 @@ struct cmdLineOpts const Real_t z[8]);*/ // lulesh-util -void -ParseCommandLineOptions(int argc, char* argv[], Int_t myRank, struct cmdLineOpts* opts); -void -VerifyAndWriteFinalOutput(Real_t elapsed_time, Domain& locDom, Int_t nx, Int_t numRanks); +void ParseCommandLineOptions(int argc, char *argv[], Int_t myRank, + struct cmdLineOpts *opts); +void VerifyAndWriteFinalOutput(Real_t elapsed_time, Domain &locDom, Int_t nx, + Int_t numRanks); // lulesh-viz -void -DumpToVisit(Domain& domain, int numFiles, int myRank, int numRanks); +void DumpToVisit(Domain &domain, int numFiles, int myRank, int numRanks); // lulesh-comm -void -CommRecv(Domain& domain, Int_t msgType, Index_t xferFields, Index_t dx, Index_t dy, - Index_t dz, bool doRecv, bool planeOnly); -void -CommSend(Domain& domain, Int_t msgType, Index_t xferFields, Domain_member* fieldData, - Index_t dx, Index_t dy, Index_t dz, bool doSend, bool planeOnly); -void -CommSBN(Domain& domain, Int_t xferFields, Domain_member* fieldData); -void -CommSyncPosVel(Domain& domain); -void -CommMonoQ(Domain& domain); +void CommRecv(Domain &domain, Int_t msgType, Index_t xferFields, Index_t dx, + Index_t dy, Index_t dz, bool doRecv, bool planeOnly); +void CommSend(Domain &domain, Int_t msgType, Index_t xferFields, + Domain_member *fieldData, Index_t dx, Index_t dy, Index_t dz, + bool doSend, bool planeOnly); +void CommSBN(Domain &domain, Int_t xferFields, Domain_member *fieldData); +void CommSyncPosVel(Domain &domain); +void CommMonoQ(Domain &domain); // lulesh-init -void -InitMeshDecomp(Int_t numRanks, Int_t myRank, Int_t* col, Int_t* row, Int_t* plane, - Int_t* side); +void InitMeshDecomp(Int_t numRanks, Int_t myRank, Int_t *col, Int_t *row, + Int_t *plane, Int_t *side); /*********************************/ /* Data structure implementation */ @@ -754,83 +757,65 @@ InitMeshDecomp(Int_t numRanks, Int_t myRank, Int_t* col, Int_t* row, Int_t* plan /* might want to add access methods so that memory can be */ /* better managed, as in luleshFT */ -template -T* -Allocate(size_t size) -{ - return static_cast(Kokkos::kokkos_malloc<>(sizeof(T) * size)); +template T *Allocate(size_t size) { + return static_cast(Kokkos::kokkos_malloc(sizeof(T) * size + 8)); } -template -void -Release(T** ptr) -{ - if(*ptr != NULL) - { - Kokkos::kokkos_free<>(*ptr); - *ptr = NULL; - } +template void Release(T **ptr) { + if (*ptr != NULL) { + Kokkos::kokkos_free(*ptr); + *ptr = NULL; + } } -struct MinFinder -{ - Real_t val; - int i; - KOKKOS_INLINE_FUNCTION +struct MinFinder { + Real_t val; + int i; + KOKKOS_INLINE_FUNCTION - MinFinder() - : val(100000000000000000000.0000) - , i(-1) - {} + MinFinder() : val(100000000000000000000.0000), i(-1) {} - KOKKOS_INLINE_FUNCTION - MinFinder(const double& val_, const int& i_) - : val(val_) - , i(i_) - {} + KOKKOS_INLINE_FUNCTION + MinFinder(const double &val_, const int &i_) : val(val_), i(i_) {} - KOKKOS_INLINE_FUNCTION - MinFinder(const MinFinder& src) - : val(src.val) - , i(src.i) - {} + KOKKOS_INLINE_FUNCTION + MinFinder(const MinFinder &src) : val(src.val), i(src.i) {} - // overloading += operator to do the max assignment - KOKKOS_INLINE_FUNCTION - void operator+=(MinFinder& src) - { - if(src.val < val) - { - val = src.val; - i = src.i; - } + // overloading += operator to do the max assignment + KOKKOS_INLINE_FUNCTION + void operator+=(MinFinder &src) { + if (src.val < val) { + val = src.val; + i = src.i; } - KOKKOS_INLINE_FUNCTION - void operator+=(const volatile MinFinder& src) volatile - { - if(src.val < val) - { - val = src.val; - i = src.i; - } + } + KOKKOS_INLINE_FUNCTION + void operator+=(const volatile MinFinder &src) volatile { + if (src.val < val) { + val = src.val; + i = src.i; } + } }; -struct reduce_double3 -{ - double x, y, z; - KOKKOS_INLINE_FUNCTION - reduce_double3() - { - x = 0.0; - y = 0.0; - z = 0.0; - } - KOKKOS_INLINE_FUNCTION - void operator+=(const reduce_double3& src) - { - x += src.x; - y += src.y; - z += src.z; - } +struct reduce_double3 { + double x, y, z; + KOKKOS_INLINE_FUNCTION + reduce_double3() { + x = 0.0; + y = 0.0; + z = 0.0; + } + KOKKOS_INLINE_FUNCTION + void operator+=(const volatile reduce_double3 &src) volatile { + x += src.x; + y += src.y; + z += src.z; + } + KOKKOS_INLINE_FUNCTION + void operator+=(const reduce_double3 &src) { + x += src.x; + y += src.y; + z += src.z; + } }; diff --git a/projects/rocprofiler-systems/examples/lulesh/lulesh_tuple.h b/projects/rocprofiler-systems/examples/lulesh/lulesh_tuple.h index e7273af967..172ec017a6 100644 --- a/projects/rocprofiler-systems/examples/lulesh/lulesh_tuple.h +++ b/projects/rocprofiler-systems/examples/lulesh/lulesh_tuple.h @@ -1,13 +1,14 @@ #if !defined(USE_MPI) -# error "You should specify USE_MPI=0 or USE_MPI=1 on the compile line" +# error "You should specify USE_MPI=0 or USE_MPI=1 on the compile line" #endif + // OpenMP will be compiled in if this flag is set to 1 AND the compiler beging // used supports it (i.e. the _OPENMP symbol is defined) #define USE_OMP 1 #if USE_MPI -# include +#include #endif #include @@ -26,110 +27,72 @@ #include //************************************************** -// Allow flexibility for arithmetic representations +// Allow flexibility for arithmetic representations //************************************************** -#define MAX(a, b) (((a) > (b)) ? (a) : (b)) +#define MAX(a, b) ( ((a) > (b)) ? (a) : (b)) + // Precision specification -typedef float real4; -typedef double real8; -typedef long double real10; // 10 bytes on x86 +typedef float real4 ; +typedef double real8 ; +typedef long double real10 ; // 10 bytes on x86 -typedef int Index_t; // array subscript and loop index -typedef real8 Real_t; // floating point representation -typedef int Int_t; // integer representation +typedef int Index_t ; // array subscript and loop index +typedef real8 Real_t ; // floating point representation +typedef int Int_t ; // integer representation -enum -{ - VolumeError = -1, - QStopError = -2 -}; +enum { VolumeError = -1, QStopError = -2 } ; -inline real4 -SQRT(real4 arg) -{ - return sqrtf(arg); -} -inline real8 -SQRT(real8 arg) -{ - return sqrt(arg); -} -inline real10 -SQRT(real10 arg) -{ - return sqrtl(arg); -} +inline real4 SQRT(real4 arg) { return sqrtf(arg) ; } +inline real8 SQRT(real8 arg) { return sqrt(arg) ; } +inline real10 SQRT(real10 arg) { return sqrtl(arg) ; } -inline real4 -CBRT(real4 arg) -{ - return cbrtf(arg); -} -inline real8 -CBRT(real8 arg) -{ - return cbrt(arg); -} -inline real10 -CBRT(real10 arg) -{ - return cbrtl(arg); -} +inline real4 CBRT(real4 arg) { return cbrtf(arg) ; } +inline real8 CBRT(real8 arg) { return cbrt(arg) ; } +inline real10 CBRT(real10 arg) { return cbrtl(arg) ; } + +inline real4 FABS(real4 arg) { return fabsf(arg) ; } +inline real8 FABS(real8 arg) { return fabs(arg) ; } +inline real10 FABS(real10 arg) { return fabsl(arg) ; } -inline real4 -FABS(real4 arg) -{ - return fabsf(arg); -} -inline real8 -FABS(real8 arg) -{ - return fabs(arg); -} -inline real10 -FABS(real10 arg) -{ - return fabsl(arg); -} // Stuff needed for boundary conditions // 2 BCs on each of 6 hexahedral faces (12 bits) -#define XI_M 0x00007 -#define XI_M_SYMM 0x00001 -#define XI_M_FREE 0x00002 -#define XI_M_COMM 0x00004 +#define XI_M 0x00007 +#define XI_M_SYMM 0x00001 +#define XI_M_FREE 0x00002 +#define XI_M_COMM 0x00004 -#define XI_P 0x00038 -#define XI_P_SYMM 0x00008 -#define XI_P_FREE 0x00010 -#define XI_P_COMM 0x00020 +#define XI_P 0x00038 +#define XI_P_SYMM 0x00008 +#define XI_P_FREE 0x00010 +#define XI_P_COMM 0x00020 -#define ETA_M 0x001c0 -#define ETA_M_SYMM 0x00040 -#define ETA_M_FREE 0x00080 -#define ETA_M_COMM 0x00100 +#define ETA_M 0x001c0 +#define ETA_M_SYMM 0x00040 +#define ETA_M_FREE 0x00080 +#define ETA_M_COMM 0x00100 -#define ETA_P 0x00e00 -#define ETA_P_SYMM 0x00200 -#define ETA_P_FREE 0x00400 -#define ETA_P_COMM 0x00800 +#define ETA_P 0x00e00 +#define ETA_P_SYMM 0x00200 +#define ETA_P_FREE 0x00400 +#define ETA_P_COMM 0x00800 -#define ZETA_M 0x07000 +#define ZETA_M 0x07000 #define ZETA_M_SYMM 0x01000 #define ZETA_M_FREE 0x02000 #define ZETA_M_COMM 0x04000 -#define ZETA_P 0x38000 +#define ZETA_P 0x38000 #define ZETA_P_SYMM 0x08000 #define ZETA_P_FREE 0x10000 #define ZETA_P_COMM 0x20000 // MPI Message Tags -#define MSG_COMM_SBN 1024 -#define MSG_SYNC_POS_VEL 2048 -#define MSG_MONOQ 3072 +#define MSG_COMM_SBN 1024 +#define MSG_SYNC_POS_VEL 2048 +#define MSG_MONOQ 3072 #define MAX_FIELDS_PER_MPI_COMM 6 @@ -137,8 +100,8 @@ FABS(real10 arg) // Assume Real_t is an "integral power of 2" bytes wide #define CACHE_COHERENCE_PAD_REAL (128 / sizeof(Real_t)) -#define CACHE_ALIGN_REAL(n) \ - (((n) + (CACHE_COHERENCE_PAD_REAL - 1)) & ~(CACHE_COHERENCE_PAD_REAL - 1)) +#define CACHE_ALIGN_REAL(n) \ + (((n) + (CACHE_COHERENCE_PAD_REAL - 1)) & ~(CACHE_COHERENCE_PAD_REAL-1)) ////////////////////////////////////////////////////// // Primary data structure @@ -163,489 +126,485 @@ FABS(real10 arg) * "Real_t &z(Index_t idx) { return m_coord[idx].z ; }" */ -class Domain -{ -public: - // Constructor - Domain(Int_t numRanks, Index_t colLoc, Index_t rowLoc, Index_t planeLoc, Index_t nx, - Int_t tp, Int_t nr, Int_t balance, Int_t cost); +class Domain { - // - // ALLOCATION - // + public: - void AllocateNodePersistent(Int_t numNode) // Node-centered - { - m_coord.resize(numNode); // coordinates + // Constructor + Domain(Int_t numRanks, Index_t colLoc, + Index_t rowLoc, Index_t planeLoc, + Index_t nx, Int_t tp, Int_t nr, Int_t balance, Int_t cost); - m_vel.resize(numNode); // velocities + // + // ALLOCATION + // - m_acc.resize(numNode); // accelerations + void AllocateNodePersistent(Int_t numNode) // Node-centered + { + m_coord.resize(numNode); // coordinates - m_force.resize(numNode); // forces + m_vel.resize(numNode); // velocities - m_nodalMass.resize(numNode); // mass - } + m_acc.resize(numNode); // accelerations - void AllocateElemPersistent(Int_t numElem) // Elem-centered - { - m_nodelist.resize(8 * numElem); + m_force.resize(numNode); // forces - // elem connectivities through face - m_faceToElem.resize(numElem); + m_nodalMass.resize(numNode); // mass + } - m_elemBC.resize(numElem); + void AllocateElemPersistent(Int_t numElem) // Elem-centered + { + m_nodelist.resize(8*numElem); - m_e.resize(numElem); + // elem connectivities through face + m_faceToElem.resize(numElem); - m_pq.resize(numElem); + m_elemBC.resize(numElem); - m_qlqq.resize(numElem); + m_e.resize(numElem); - m_vol.resize(numElem); + m_pq.resize(numElem); - m_delv.resize(numElem); - m_vdov.resize(numElem); + m_qlqq.resize(numElem); - m_arealg.resize(numElem); + m_vol.resize(numElem); - m_ss.resize(numElem); + m_delv.resize(numElem); + m_vdov.resize(numElem); - m_elemMass.resize(numElem); - } + m_arealg.resize(numElem); - void AllocateGradients(Int_t numElem, Int_t allElem) - { - // Position gradients - m_delx_xi.resize(numElem); - m_delx_eta.resize(numElem); - m_delx_zeta.resize(numElem); + m_ss.resize(numElem); - // Velocity gradients - m_delv_xi.resize(allElem); - m_delv_eta.resize(allElem); - m_delv_zeta.resize(allElem); - } + m_elemMass.resize(numElem); + } - void DeallocateGradients() - { - m_delx_zeta.clear(); - m_delx_eta.clear(); - m_delx_xi.clear(); + void AllocateGradients(Int_t numElem, Int_t allElem) + { + // Position gradients + m_delx_xi.resize(numElem) ; + m_delx_eta.resize(numElem) ; + m_delx_zeta.resize(numElem) ; - m_delv_zeta.clear(); - m_delv_eta.clear(); - m_delv_xi.clear(); - } + // Velocity gradients + m_delv_xi.resize(allElem) ; + m_delv_eta.resize(allElem); + m_delv_zeta.resize(allElem) ; + } - void AllocateStrains(Int_t numElem) - { - m_dxx.resize(numElem); - m_dyy.resize(numElem); - m_dzz.resize(numElem); - } + void DeallocateGradients() + { + m_delx_zeta.clear() ; + m_delx_eta.clear() ; + m_delx_xi.clear() ; - void DeallocateStrains() - { - m_dzz.clear(); - m_dyy.clear(); - m_dxx.clear(); - } + m_delv_zeta.clear() ; + m_delv_eta.clear() ; + m_delv_xi.clear() ; + } - // - // ACCESSORS - // + void AllocateStrains(Int_t numElem) + { + m_dxx.resize(numElem) ; + m_dyy.resize(numElem) ; + m_dzz.resize(numElem) ; + } - // Node-centered + void DeallocateStrains() + { + m_dzz.clear() ; + m_dyy.clear() ; + m_dxx.clear() ; + } + + // + // ACCESSORS + // - // Nodal coordinates - Real_t& x(Index_t idx) { return m_coord[idx].x; } - Real_t& y(Index_t idx) { return m_coord[idx].y; } - Real_t& z(Index_t idx) { return m_coord[idx].z; } + // Node-centered - // Nodal velocities - Real_t& xd(Index_t idx) { return m_vel[idx].x; } - Real_t& yd(Index_t idx) { return m_vel[idx].y; } - Real_t& zd(Index_t idx) { return m_vel[idx].z; } + // Nodal coordinates + Real_t& x(Index_t idx) { return m_coord[idx].x ; } + Real_t& y(Index_t idx) { return m_coord[idx].y ; } + Real_t& z(Index_t idx) { return m_coord[idx].z ; } - // Nodal accelerations - Real_t& xdd(Index_t idx) { return m_acc[idx].x; } - Real_t& ydd(Index_t idx) { return m_acc[idx].y; } - Real_t& zdd(Index_t idx) { return m_acc[idx].z; } + // Nodal velocities + Real_t& xd(Index_t idx) { return m_vel[idx].x ; } + Real_t& yd(Index_t idx) { return m_vel[idx].y ; } + Real_t& zd(Index_t idx) { return m_vel[idx].z ; } - // Nodal forces - Real_t& fx(Index_t idx) { return m_force[idx].x; } - Real_t& fy(Index_t idx) { return m_force[idx].y; } - Real_t& fz(Index_t idx) { return m_force[idx].z; } + // Nodal accelerations + Real_t& xdd(Index_t idx) { return m_acc[idx].x ; } + Real_t& ydd(Index_t idx) { return m_acc[idx].y ; } + Real_t& zdd(Index_t idx) { return m_acc[idx].z ; } - // Nodal mass - Real_t& nodalMass(Index_t idx) { return m_nodalMass[idx]; } + // Nodal forces + Real_t& fx(Index_t idx) { return m_force[idx].x ; } + Real_t& fy(Index_t idx) { return m_force[idx].y ; } + Real_t& fz(Index_t idx) { return m_force[idx].z ; } - // Nodes on symmertry planes - Index_t symmX(Index_t idx) { return m_symmX[idx]; } - Index_t symmY(Index_t idx) { return m_symmY[idx]; } - Index_t symmZ(Index_t idx) { return m_symmZ[idx]; } - bool symmXempty() { return m_symmX.empty(); } - bool symmYempty() { return m_symmY.empty(); } - bool symmZempty() { return m_symmZ.empty(); } + // Nodal mass + Real_t& nodalMass(Index_t idx) { return m_nodalMass[idx] ; } - // - // Element-centered - // - Index_t& regElemSize(Index_t idx) { return m_regElemSize[idx]; } - Index_t& regNumList(Index_t idx) { return m_regNumList[idx]; } - Index_t* regNumList() { return &m_regNumList[0]; } - Index_t* regElemlist(Int_t r) { return m_regElemlist[r]; } - Index_t& regElemlist(Int_t r, Index_t idx) { return m_regElemlist[r][idx]; } + // Nodes on symmertry planes + Index_t symmX(Index_t idx) { return m_symmX[idx] ; } + Index_t symmY(Index_t idx) { return m_symmY[idx] ; } + Index_t symmZ(Index_t idx) { return m_symmZ[idx] ; } + bool symmXempty() { return m_symmX.empty(); } + bool symmYempty() { return m_symmY.empty(); } + bool symmZempty() { return m_symmZ.empty(); } - Index_t* nodelist(Index_t idx) { return &m_nodelist[Index_t(8) * idx]; } + // + // Element-centered + // + Index_t& regElemSize(Index_t idx) { return m_regElemSize[idx] ; } + Index_t& regNumList(Index_t idx) { return m_regNumList[idx] ; } + Index_t* regNumList() { return &m_regNumList[0] ; } + Index_t* regElemlist(Int_t r) { return m_regElemlist[r] ; } + Index_t& regElemlist(Int_t r, Index_t idx) { return m_regElemlist[r][idx] ; } - // elem connectivities through face - Index_t& lxim(Index_t idx) { return m_faceToElem[idx].lxim; } - Index_t& lxip(Index_t idx) { return m_faceToElem[idx].lxip; } - Index_t& letam(Index_t idx) { return m_faceToElem[idx].letam; } - Index_t& letap(Index_t idx) { return m_faceToElem[idx].letap; } - Index_t& lzetam(Index_t idx) { return m_faceToElem[idx].lzetam; } - Index_t& lzetap(Index_t idx) { return m_faceToElem[idx].lzetap; } + Index_t* nodelist(Index_t idx) { return &m_nodelist[Index_t(8)*idx] ; } - // elem face symm/free-surface flag - Int_t& elemBC(Index_t idx) { return m_elemBC[idx]; } + // elem connectivities through face + Index_t& lxim(Index_t idx) { return m_faceToElem[idx].lxim ; } + Index_t& lxip(Index_t idx) { return m_faceToElem[idx].lxip ; } + Index_t& letam(Index_t idx) { return m_faceToElem[idx].letam ; } + Index_t& letap(Index_t idx) { return m_faceToElem[idx].letap ; } + Index_t& lzetam(Index_t idx) { return m_faceToElem[idx].lzetam ; } + Index_t& lzetap(Index_t idx) { return m_faceToElem[idx].lzetap ; } - // Principal strains - temporary - Real_t& dxx(Index_t idx) { return m_dxx[idx]; } - Real_t& dyy(Index_t idx) { return m_dyy[idx]; } - Real_t& dzz(Index_t idx) { return m_dzz[idx]; } + // elem face symm/free-surface flag + Int_t& elemBC(Index_t idx) { return m_elemBC[idx] ; } - // Velocity gradient - temporary - Real_t& delv_xi(Index_t idx) { return m_delv_xi[idx]; } - Real_t& delv_eta(Index_t idx) { return m_delv_eta[idx]; } - Real_t& delv_zeta(Index_t idx) { return m_delv_zeta[idx]; } + // Principal strains - temporary + Real_t& dxx(Index_t idx) { return m_dxx[idx] ; } + Real_t& dyy(Index_t idx) { return m_dyy[idx] ; } + Real_t& dzz(Index_t idx) { return m_dzz[idx] ; } - // Position gradient - temporary - Real_t& delx_xi(Index_t idx) { return m_delx_xi[idx]; } - Real_t& delx_eta(Index_t idx) { return m_delx_eta[idx]; } - Real_t& delx_zeta(Index_t idx) { return m_delx_zeta[idx]; } + // Velocity gradient - temporary + Real_t& delv_xi(Index_t idx) { return m_delv_xi[idx] ; } + Real_t& delv_eta(Index_t idx) { return m_delv_eta[idx] ; } + Real_t& delv_zeta(Index_t idx) { return m_delv_zeta[idx] ; } - // Energy - Real_t& e(Index_t idx) { return m_e[idx]; } + // Position gradient - temporary + Real_t& delx_xi(Index_t idx) { return m_delx_xi[idx] ; } + Real_t& delx_eta(Index_t idx) { return m_delx_eta[idx] ; } + Real_t& delx_zeta(Index_t idx) { return m_delx_zeta[idx] ; } - // Pressure - Real_t& p(Index_t idx) { return m_pq[idx].p; } + // Energy + Real_t& e(Index_t idx) { return m_e[idx] ; } - // Artificial viscosity - Real_t& q(Index_t idx) { return m_pq[idx].q; } + // Pressure + Real_t& p(Index_t idx) { return m_pq[idx].p ; } - // Linear term for q - Real_t& ql(Index_t idx) { return m_qlqq[idx].ql; } - // Quadratic term for q - Real_t& qq(Index_t idx) { return m_qlqq[idx].qq; } + // Artificial viscosity + Real_t& q(Index_t idx) { return m_pq[idx].q ; } - Real_t& delv(Index_t idx) { return m_delv[idx]; } + // Linear term for q + Real_t& ql(Index_t idx) { return m_qlqq[idx].ql ; } + // Quadratic term for q + Real_t& qq(Index_t idx) { return m_qlqq[idx].qq ; } - // Relative volume - Real_t& v(Index_t idx) { return m_vol[idx].v; } - // Reference volume - Real_t& volo(Index_t idx) { return m_vol[idx].volo; } + Real_t& delv(Index_t idx) { return m_delv[idx] ; } - // volume derivative over volume - Real_t& vdov(Index_t idx) { return m_vdov[idx]; } + // Relative volume + Real_t& v(Index_t idx) { return m_vol[idx].v ; } + // Reference volume + Real_t& volo(Index_t idx) { return m_vol[idx].volo ; } - // Element characteristic length - Real_t& arealg(Index_t idx) { return m_arealg[idx]; } + // volume derivative over volume + Real_t& vdov(Index_t idx) { return m_vdov[idx] ; } - // Sound speed - Real_t& ss(Index_t idx) { return m_ss[idx]; } + // Element characteristic length + Real_t& arealg(Index_t idx) { return m_arealg[idx] ; } - // Element mass - Real_t& elemMass(Index_t idx) { return m_elemMass[idx]; } + // Sound speed + Real_t& ss(Index_t idx) { return m_ss[idx] ; } - Index_t nodeElemCount(Index_t idx) - { - return m_nodeElemStart[idx + 1] - m_nodeElemStart[idx]; - } + // Element mass + Real_t& elemMass(Index_t idx) { return m_elemMass[idx] ; } - Index_t* nodeElemCornerList(Index_t idx) - { - return &m_nodeElemCornerList[m_nodeElemStart[idx]]; - } + Index_t nodeElemCount(Index_t idx) + { return m_nodeElemStart[idx+1] - m_nodeElemStart[idx] ; } - // Parameters + Index_t *nodeElemCornerList(Index_t idx) + { return &m_nodeElemCornerList[m_nodeElemStart[idx]] ; } - // Cutoffs - Real_t u_cut() const { return m_u_cut; } - Real_t e_cut() const { return m_e_cut; } - Real_t p_cut() const { return m_p_cut; } - Real_t q_cut() const { return m_q_cut; } - Real_t v_cut() const { return m_v_cut; } + // Parameters - // Other constants (usually are settable via input file in real codes) - Real_t hgcoef() const { return m_hgcoef; } - Real_t qstop() const { return m_qstop; } - Real_t monoq_max_slope() const { return m_monoq_max_slope; } - Real_t monoq_limiter_mult() const { return m_monoq_limiter_mult; } - Real_t ss4o3() const { return m_ss4o3; } - Real_t qlc_monoq() const { return m_qlc_monoq; } - Real_t qqc_monoq() const { return m_qqc_monoq; } - Real_t qqc() const { return m_qqc; } + // Cutoffs + Real_t u_cut() const { return m_u_cut ; } + Real_t e_cut() const { return m_e_cut ; } + Real_t p_cut() const { return m_p_cut ; } + Real_t q_cut() const { return m_q_cut ; } + Real_t v_cut() const { return m_v_cut ; } - Real_t eosvmax() const { return m_eosvmax; } - Real_t eosvmin() const { return m_eosvmin; } - Real_t pmin() const { return m_pmin; } - Real_t emin() const { return m_emin; } - Real_t dvovmax() const { return m_dvovmax; } - Real_t refdens() const { return m_refdens; } + // Other constants (usually are settable via input file in real codes) + Real_t hgcoef() const { return m_hgcoef ; } + Real_t qstop() const { return m_qstop ; } + Real_t monoq_max_slope() const { return m_monoq_max_slope ; } + Real_t monoq_limiter_mult() const { return m_monoq_limiter_mult ; } + Real_t ss4o3() const { return m_ss4o3 ; } + Real_t qlc_monoq() const { return m_qlc_monoq ; } + Real_t qqc_monoq() const { return m_qqc_monoq ; } + Real_t qqc() const { return m_qqc ; } - // Timestep controls, etc... - Real_t& time() { return m_time; } - Real_t& deltatime() { return m_deltatime; } - Real_t& deltatimemultlb() { return m_deltatimemultlb; } - Real_t& deltatimemultub() { return m_deltatimemultub; } - Real_t& stoptime() { return m_stoptime; } - Real_t& dtcourant() { return m_dtcourant; } - Real_t& dthydro() { return m_dthydro; } - Real_t& dtmax() { return m_dtmax; } - Real_t& dtfixed() { return m_dtfixed; } + Real_t eosvmax() const { return m_eosvmax ; } + Real_t eosvmin() const { return m_eosvmin ; } + Real_t pmin() const { return m_pmin ; } + Real_t emin() const { return m_emin ; } + Real_t dvovmax() const { return m_dvovmax ; } + Real_t refdens() const { return m_refdens ; } - Int_t& cycle() { return m_cycle; } - Index_t& numRanks() { return m_numRanks; } + // Timestep controls, etc... + Real_t& time() { return m_time ; } + Real_t& deltatime() { return m_deltatime ; } + Real_t& deltatimemultlb() { return m_deltatimemultlb ; } + Real_t& deltatimemultub() { return m_deltatimemultub ; } + Real_t& stoptime() { return m_stoptime ; } + Real_t& dtcourant() { return m_dtcourant ; } + Real_t& dthydro() { return m_dthydro ; } + Real_t& dtmax() { return m_dtmax ; } + Real_t& dtfixed() { return m_dtfixed ; } - Index_t& colLoc() { return m_colLoc; } - Index_t& rowLoc() { return m_rowLoc; } - Index_t& planeLoc() { return m_planeLoc; } - Index_t& tp() { return m_tp; } + Int_t& cycle() { return m_cycle ; } + Index_t& numRanks() { return m_numRanks ; } - Index_t& sizeX() { return m_sizeX; } - Index_t& sizeY() { return m_sizeY; } - Index_t& sizeZ() { return m_sizeZ; } - Index_t& numReg() { return m_numReg; } - Int_t& cost() { return m_cost; } - Index_t& numElem() { return m_numElem; } - Index_t& numNode() { return m_numNode; } + Index_t& colLoc() { return m_colLoc ; } + Index_t& rowLoc() { return m_rowLoc ; } + Index_t& planeLoc() { return m_planeLoc ; } + Index_t& tp() { return m_tp ; } - Index_t& maxPlaneSize() { return m_maxPlaneSize; } - Index_t& maxEdgeSize() { return m_maxEdgeSize; } + Index_t& sizeX() { return m_sizeX ; } + Index_t& sizeY() { return m_sizeY ; } + Index_t& sizeZ() { return m_sizeZ ; } + Index_t& numReg() { return m_numReg ; } + Int_t& cost() { return m_cost ; } + Index_t& numElem() { return m_numElem ; } + Index_t& numNode() { return m_numNode ; } + + Index_t& maxPlaneSize() { return m_maxPlaneSize ; } + Index_t& maxEdgeSize() { return m_maxEdgeSize ; } + + // + // MPI-Related additional data + // - // - // MPI-Related additional data - // - -#if USE_MPI - // Communication Work space - Real_t* commDataSend; - Real_t* commDataRecv; - - // Maximum number of block neighbors - MPI_Request recvRequest[26]; // 6 faces + 12 edges + 8 corners - MPI_Request sendRequest[26]; // 6 faces + 12 edges + 8 corners +#if USE_MPI + // Communication Work space + Real_t *commDataSend ; + Real_t *commDataRecv ; + + // Maximum number of block neighbors + MPI_Request recvRequest[26] ; // 6 faces + 12 edges + 8 corners + MPI_Request sendRequest[26] ; // 6 faces + 12 edges + 8 corners #endif -private: - void BuildMesh(Int_t nx, Int_t edgeNodes, Int_t edgeElems); - void SetupThreadSupportStructures(); - void CreateRegionIndexSets(Int_t nreg, Int_t balance); - void SetupCommBuffers(Int_t edgeNodes); - void SetupSymmetryPlanes(Int_t edgeNodes); - void SetupElementConnectivities(Int_t edgeElems); - void SetupBoundaryConditions(Int_t edgeElems); + private: - // - // IMPLEMENTATION - // + void BuildMesh(Int_t nx, Int_t edgeNodes, Int_t edgeElems); + void SetupThreadSupportStructures(); + void CreateRegionIndexSets(Int_t nreg, Int_t balance); + void SetupCommBuffers(Int_t edgeNodes); + void SetupSymmetryPlanes(Int_t edgeNodes); + void SetupElementConnectivities(Int_t edgeElems); + void SetupBoundaryConditions(Int_t edgeElems); - /* Node-centered */ + // + // IMPLEMENTATION + // - struct Tuple3 - { - Real_t x, y, z; - }; + /* Node-centered */ - Kokkos::vector m_coord; /* coordinates */ + struct Tuple3 { + Real_t x, y, z ; + } ; - Kokkos::vector m_vel; /* velocities */ + Kokkos::View m_coord ; /* coordinates */ - Kokkos::vector m_acc; /* accelerations */ + Kokkos::View m_vel ; /* velocities */ - Kokkos::vector m_force; /* forces */ + Kokkos::View m_acc ; /* accelerations */ - Kokkos::vector m_nodalMass; /* mass */ + Kokkos::View m_force ; /* forces */ - Kokkos::vector m_symmX; /* symmetry plane nodesets */ - Kokkos::vector m_symmY; - Kokkos::vector m_symmZ; + Kokkos::View m_nodalMass ; /* mass */ - // Element-centered + Kokkos::View m_symmX ; /* symmetry plane nodesets */ + Kokkos::View m_symmY ; + Kokkos::View m_symmZ ; - // Region information - Int_t m_numReg; - Int_t m_cost; // imbalance cost - Index_t* m_regElemSize; // Size of region sets - Index_t* m_regNumList; // Region number per domain element - Index_t** m_regElemlist; // region indexset + // Element-centered - Kokkos::vector m_nodelist; /* elemToNode connectivity */ + // Region information + Int_t m_numReg ; + Int_t m_cost; //imbalance cost + Index_t *m_regElemSize ; // Size of region sets + Index_t *m_regNumList ; // Region number per domain element + Index_t **m_regElemlist ; // region indexset - struct FaceElemConn - { - Index_t lxim, lxip, letam, letap, lzetam, lzetap; - }; + Kokkos::View m_nodelist ; /* elemToNode connectivity */ - Kokkos::vector m_faceToElem; /* element conn across faces */ + struct FaceElemConn { + Index_t lxim, lxip, letam, letap, lzetam, lzetap ; + } ; - Kokkos::vector m_elemBC; /* symmetry/free-surface flags for each elem face */ + Kokkos::View m_faceToElem ; /* element conn across faces */ - Kokkos::vector m_dxx; /* principal strains -- temporary */ - Kokkos::vector m_dyy; - Kokkos::vector m_dzz; + Kokkos::View m_elemBC ; /* symmetry/free-surface flags for each elem face */ - Kokkos::vector m_delv_xi; /* velocity gradient -- temporary */ - Kokkos::vector m_delv_eta; - Kokkos::vector m_delv_zeta; + Kokkos::View m_dxx ; /* principal strains -- temporary */ + Kokkos::View m_dyy ; + Kokkos::View m_dzz ; - Kokkos::vector m_delx_xi; /* coordinate gradient -- temporary */ - Kokkos::vector m_delx_eta; - Kokkos::vector m_delx_zeta; + Kokkos::View m_delv_xi ; /* velocity gradient -- temporary */ + Kokkos::View m_delv_eta ; + Kokkos::View m_delv_zeta ; - Kokkos::vector m_e; /* energy */ + Kokkos::View m_delx_xi ; /* coordinate gradient -- temporary */ + Kokkos::View m_delx_eta ; + Kokkos::View m_delx_zeta ; + + Kokkos::View m_e ; /* energy */ - struct Pcomponents - { - Real_t p, q; - }; + struct Pcomponents { + Real_t p, q ; + } ; - Kokkos::vector m_pq; /* pressure and artificial viscosity */ + Kokkos::View m_pq ; /* pressure and artificial viscosity */ - struct Qcomponents - { - Real_t ql, qq; - }; + struct Qcomponents { + Real_t ql, qq ; + } ; - Kokkos::vector m_qlqq; /* linear and quadratic terms for q */ + Kokkos::View m_qlqq ; /* linear and quadratic terms for q */ - struct Volume - { - Real_t v, volo; - }; + struct Volume { + Real_t v, volo ; + } ; - Kokkos::vector m_vol; /* relative and reference volume */ + Kokkos::View m_vol ; /* relative and reference volume */ - Kokkos::vector m_vnew; /* new relative volume -- temporary */ - Kokkos::vector m_delv; /* m_vnew - m_v */ - Kokkos::vector m_vdov; /* volume derivative over volume */ + Kokkos::View m_vnew ; /* new relative volume -- temporary */ + Kokkos::View m_delv ; /* m_vnew - m_v */ + Kokkos::View m_vdov ; /* volume derivative over volume */ - Kokkos::vector m_arealg; /* characteristic length of an element */ + Kokkos::View m_arealg ; /* characteristic length of an element */ + + Kokkos::View m_ss ; /* "sound speed" */ - Kokkos::vector m_ss; /* "sound speed" */ + Kokkos::View m_elemMass ; /* mass */ - Kokkos::vector m_elemMass; /* mass */ + // Cutoffs (treat as constants) + const Real_t m_e_cut ; // energy tolerance + const Real_t m_p_cut ; // pressure tolerance + const Real_t m_q_cut ; // q tolerance + const Real_t m_v_cut ; // relative volume tolerance + const Real_t m_u_cut ; // velocity tolerance - // Cutoffs (treat as constants) - const Real_t m_e_cut; // energy tolerance - const Real_t m_p_cut; // pressure tolerance - const Real_t m_q_cut; // q tolerance - const Real_t m_v_cut; // relative volume tolerance - const Real_t m_u_cut; // velocity tolerance + // Other constants (usually setable, but hardcoded in this proxy app) - // Other constants (usually setable, but hardcoded in this proxy app) + const Real_t m_hgcoef ; // hourglass control + const Real_t m_ss4o3 ; + const Real_t m_qstop ; // excessive q indicator + const Real_t m_monoq_max_slope ; + const Real_t m_monoq_limiter_mult ; + const Real_t m_qlc_monoq ; // linear term coef for q + const Real_t m_qqc_monoq ; // quadratic term coef for q + const Real_t m_qqc ; + const Real_t m_eosvmax ; + const Real_t m_eosvmin ; + const Real_t m_pmin ; // pressure floor + const Real_t m_emin ; // energy floor + const Real_t m_dvovmax ; // maximum allowable volume change + const Real_t m_refdens ; // reference density - const Real_t m_hgcoef; // hourglass control - const Real_t m_ss4o3; - const Real_t m_qstop; // excessive q indicator - const Real_t m_monoq_max_slope; - const Real_t m_monoq_limiter_mult; - const Real_t m_qlc_monoq; // linear term coef for q - const Real_t m_qqc_monoq; // quadratic term coef for q - const Real_t m_qqc; - const Real_t m_eosvmax; - const Real_t m_eosvmin; - const Real_t m_pmin; // pressure floor - const Real_t m_emin; // energy floor - const Real_t m_dvovmax; // maximum allowable volume change - const Real_t m_refdens; // reference density + // Variables to keep track of timestep, simulation time, and cycle + Real_t m_dtcourant ; // courant constraint + Real_t m_dthydro ; // volume change constraint + Int_t m_cycle ; // iteration count for simulation + Real_t m_dtfixed ; // fixed time increment + Real_t m_time ; // current time + Real_t m_deltatime ; // variable time increment + Real_t m_deltatimemultlb ; + Real_t m_deltatimemultub ; + Real_t m_dtmax ; // maximum allowable time increment + Real_t m_stoptime ; // end time for simulation - // Variables to keep track of timestep, simulation time, and cycle - Real_t m_dtcourant; // courant constraint - Real_t m_dthydro; // volume change constraint - Int_t m_cycle; // iteration count for simulation - Real_t m_dtfixed; // fixed time increment - Real_t m_time; // current time - Real_t m_deltatime; // variable time increment - Real_t m_deltatimemultlb; - Real_t m_deltatimemultub; - Real_t m_dtmax; // maximum allowable time increment - Real_t m_stoptime; // end time for simulation - Int_t m_numRanks; + Int_t m_numRanks ; - Index_t m_colLoc; - Index_t m_rowLoc; - Index_t m_planeLoc; - Index_t m_tp; + Index_t m_colLoc ; + Index_t m_rowLoc ; + Index_t m_planeLoc ; + Index_t m_tp ; - Index_t m_sizeX; - Index_t m_sizeY; - Index_t m_sizeZ; - Index_t m_numElem; - Index_t m_numNode; + Index_t m_sizeX ; + Index_t m_sizeY ; + Index_t m_sizeZ ; + Index_t m_numElem ; + Index_t m_numNode ; - Index_t m_maxPlaneSize; - Index_t m_maxEdgeSize; + Index_t m_maxPlaneSize ; + Index_t m_maxEdgeSize ; - // OMP hack - Index_t* m_nodeElemStart; - Index_t* m_nodeElemCornerList; + // OMP hack + Index_t *m_nodeElemStart ; + Index_t *m_nodeElemCornerList ; - // Used in setup - Index_t m_rowMin, m_rowMax; - Index_t m_colMin, m_colMax; - Index_t m_planeMin, m_planeMax; + // Used in setup + Index_t m_rowMin, m_rowMax; + Index_t m_colMin, m_colMax; + Index_t m_planeMin, m_planeMax ; + +} ; + +typedef Real_t &(Domain::* Domain_member )(Index_t) ; + +struct cmdLineOpts { + Int_t its; // -i + Int_t nx; // -s + Int_t numReg; // -r + Int_t numFiles; // -f + Int_t showProg; // -p + Int_t quiet; // -q + Int_t viz; // -v + Int_t cost; // -c + Int_t balance; // -b }; -typedef Real_t& (Domain::*Domain_member)(Index_t); -struct cmdLineOpts -{ - Int_t its; // -i - Int_t nx; // -s - Int_t numReg; // -r - Int_t numFiles; // -f - Int_t showProg; // -p - Int_t quiet; // -q - Int_t viz; // -v - Int_t cost; // -c - Int_t balance; // -b -}; // Function Prototypes // lulesh-par -Real_t -CalcElemVolume(const Real_t x[8], const Real_t y[8], const Real_t z[8]); +Real_t CalcElemVolume( const Real_t x[8], + const Real_t y[8], + const Real_t z[8]); // lulesh-util -void -ParseCommandLineOptions(int argc, char* argv[], Int_t myRank, struct cmdLineOpts* opts); -void -VerifyAndWriteFinalOutput(Real_t elapsed_time, Domain& locDom, Int_t nx, Int_t numRanks); +void ParseCommandLineOptions(int argc, char *argv[], + Int_t myRank, struct cmdLineOpts *opts); +void VerifyAndWriteFinalOutput(Real_t elapsed_time, + Domain& locDom, + Int_t nx, + Int_t numRanks); // lulesh-viz -void -DumpToVisit(Domain& domain, int numFiles, int myRank, int numRanks); +void DumpToVisit(Domain& domain, int numFiles, int myRank, int numRanks); // lulesh-comm -void -CommRecv(Domain& domain, Int_t msgType, Index_t xferFields, Index_t dx, Index_t dy, - Index_t dz, bool doRecv, bool planeOnly); -void -CommSend(Domain& domain, Int_t msgType, Index_t xferFields, Domain_member* fieldData, - Index_t dx, Index_t dy, Index_t dz, bool doSend, bool planeOnly); -void -CommSBN(Domain& domain, Int_t xferFields, Domain_member* fieldData); -void -CommSyncPosVel(Domain& domain); -void -CommMonoQ(Domain& domain); +void CommRecv(Domain& domain, Int_t msgType, Index_t xferFields, + Index_t dx, Index_t dy, Index_t dz, + bool doRecv, bool planeOnly); +void CommSend(Domain& domain, Int_t msgType, + Index_t xferFields, Domain_member *fieldData, + Index_t dx, Index_t dy, Index_t dz, + bool doSend, bool planeOnly); +void CommSBN(Domain& domain, Int_t xferFields, Domain_member *fieldData); +void CommSyncPosVel(Domain& domain); +void CommMonoQ(Domain& domain); // lulesh-init -void -InitMeshDecomp(Int_t numRanks, Int_t myRank, Int_t* col, Int_t* row, Int_t* plane, - Int_t* side); +void InitMeshDecomp(Int_t numRanks, Int_t myRank, + Int_t *col, Int_t *row, Int_t *plane, Int_t *side); diff --git a/projects/rocprofiler-systems/examples/transpose/CMakeLists.txt b/projects/rocprofiler-systems/examples/transpose/CMakeLists.txt index c918563590..2434fe37bb 100644 --- a/projects/rocprofiler-systems/examples/transpose/CMakeLists.txt +++ b/projects/rocprofiler-systems/examples/transpose/CMakeLists.txt @@ -3,6 +3,7 @@ cmake_minimum_required(VERSION 3.15 FATAL_ERROR) project(omnitrace-transpose LANGUAGES CXX) find_program(HIPCC_EXECUTABLE NAMES hipcc) +mark_as_advanced(HIPCC_EXECUTABLE) if(NOT HIPCC_EXECUTABLE) message(AUTHOR_WARNING "hipcc could not be found. Cannot build transpose target") diff --git a/projects/rocprofiler-systems/external/perfetto b/projects/rocprofiler-systems/external/perfetto index dd1f2f378f..0551922853 160000 --- a/projects/rocprofiler-systems/external/perfetto +++ b/projects/rocprofiler-systems/external/perfetto @@ -1 +1 @@ -Subproject commit dd1f2f378fe2f292f78af922828f93a9f101d373 +Subproject commit 0551922853f0494336e1abd82de66de829c95b4e diff --git a/projects/rocprofiler-systems/external/timemory b/projects/rocprofiler-systems/external/timemory index ccc83e80c6..110b907b35 160000 --- a/projects/rocprofiler-systems/external/timemory +++ b/projects/rocprofiler-systems/external/timemory @@ -1 +1 @@ -Subproject commit ccc83e80c6b56b9404bdbc8c341dcbb621084ef6 +Subproject commit 110b907b352d9b51df6b439879ce0439503aa073 diff --git a/projects/rocprofiler-systems/include/library.hpp b/projects/rocprofiler-systems/include/library.hpp index f83b41c8a2..e2cae066d9 100644 --- a/projects/rocprofiler-systems/include/library.hpp +++ b/projects/rocprofiler-systems/include/library.hpp @@ -55,8 +55,6 @@ add_critical_trace(int64_t _tid, size_t _cpu_cid, size_t _gpu_cid, size_t _paren int64_t _ts_beg, int64_t _ts_val, size_t _hash, uint16_t _depth, uint16_t _prio = 0) { - if(!get_use_critical_trace()) return; - // clang-format off // these are used to create unique type mutexes struct critical_insert {}; diff --git a/projects/rocprofiler-systems/include/library/api.hpp b/projects/rocprofiler-systems/include/library/api.hpp index b4470c66ff..a2ef6b2e6e 100644 --- a/projects/rocprofiler-systems/include/library/api.hpp +++ b/projects/rocprofiler-systems/include/library/api.hpp @@ -29,12 +29,25 @@ // forward decl of the API extern "C" { + /// handles configuration logic + void omnitrace_init_library(void) TIMEMORY_VISIBILITY("default"); + + /// starts gotcha wrappers + void omnitrace_init(const char*, bool, const char*) TIMEMORY_VISIBILITY("default"); + + /// shuts down all tooling and generates output + void omnitrace_finalize(void) TIMEMORY_VISIBILITY("default"); + + /// sets an environment variable + void omnitrace_set_env(const char* env_name, const char* env_val) + TIMEMORY_VISIBILITY("default"); + + /// sets whether MPI should be used + void omnitrace_set_mpi(bool use, bool attached) TIMEMORY_VISIBILITY("default"); + + /// starts an instrumentation region void omnitrace_push_trace(const char* name) TIMEMORY_VISIBILITY("default"); + + /// stops an instrumentation region void omnitrace_pop_trace(const char* name) TIMEMORY_VISIBILITY("default"); - void omnitrace_trace_init(const char*, bool, const char*) - TIMEMORY_VISIBILITY("default"); - void omnitrace_trace_finalize(void) TIMEMORY_VISIBILITY("default"); - void omnitrace_trace_set_env(const char* env_name, const char* env_val) - TIMEMORY_VISIBILITY("default"); - void omnitrace_trace_set_mpi(bool use, bool attached) TIMEMORY_VISIBILITY("default"); } diff --git a/projects/rocprofiler-systems/include/library/common.hpp b/projects/rocprofiler-systems/include/library/common.hpp index 7713b6670a..8b84ce7142 100644 --- a/projects/rocprofiler-systems/include/library/common.hpp +++ b/projects/rocprofiler-systems/include/library/common.hpp @@ -43,6 +43,7 @@ TIMEMORY_DEFINE_NS_API(api, omnitrace) TIMEMORY_DEFINE_NS_API(api, sampling) +TIMEMORY_DEFINE_NS_API(api, rocm_smi) namespace omnitrace { diff --git a/projects/rocprofiler-systems/include/library/components/backtrace.hpp b/projects/rocprofiler-systems/include/library/components/backtrace.hpp index b55d173031..30a87b11bb 100644 --- a/projects/rocprofiler-systems/include/library/components/backtrace.hpp +++ b/projects/rocprofiler-systems/include/library/components/backtrace.hpp @@ -91,6 +91,7 @@ struct backtrace private: int64_t m_tid = 0; int64_t m_thr_cpu_ts = 0; + int64_t m_mem_peak = 0; size_t m_size = 0; time_point_type m_ts = {}; data_t m_data = {}; diff --git a/projects/rocprofiler-systems/include/library/components/fwd.hpp b/projects/rocprofiler-systems/include/library/components/fwd.hpp index 2f51fc6ae2..dcc769b452 100644 --- a/projects/rocprofiler-systems/include/library/components/fwd.hpp +++ b/projects/rocprofiler-systems/include/library/components/fwd.hpp @@ -28,6 +28,7 @@ #include #include #include +#include TIMEMORY_DECLARE_COMPONENT(roctracer) @@ -46,9 +47,21 @@ struct backtrace_cpu_clock {}; struct backtrace_fraction {}; +struct backtrace_gpu_busy +{}; +struct backtrace_gpu_temp +{}; +struct backtrace_gpu_power +{}; +struct backtrace_gpu_memory +{}; using sampling_wall_clock = data_tracker; using sampling_cpu_clock = data_tracker; using sampling_percent = data_tracker; +using sampling_gpu_busy = data_tracker; +using sampling_gpu_temp = data_tracker; +using sampling_gpu_power = data_tracker; +using sampling_gpu_memory = data_tracker; using roctracer = tim::component::roctracer; } // namespace component } // namespace omnitrace @@ -59,7 +72,7 @@ TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, component::roctracer, false_type) #if !defined(TIMEMORY_USE_LIBUNWIND) TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, omnitrace::api::sampling, false_type) -TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, omnitrace::sampling::backtrace, false_type) +TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, omnitrace::component::backtrace, false_type) TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, omnitrace::component::sampling_wall_clock, false_type) TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, omnitrace::component::sampling_cpu_clock, @@ -68,6 +81,17 @@ TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, omnitrace::component::sampling_perc false_type) #endif +#if !defined(TIMEMORY_USE_LIBUNWIND) || !defined(OMNITRACE_USE_ROCM_SMI) +TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, omnitrace::component::sampling_gpu_busy, + false_type) +TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, omnitrace::component::sampling_gpu_temp, + false_type) +TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, omnitrace::component::sampling_gpu_power, + false_type) +TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, omnitrace::component::sampling_gpu_memory, + false_type) +#endif + TIMEMORY_PROPERTY_SPECIALIZATION(omnitrace::component::omnitrace, OMNITRACE_COMPONENT, "omnitrace", "omnitrace_component") TIMEMORY_PROPERTY_SPECIALIZATION(omnitrace::component::roctracer, OMNITRACE_ROCTRACER, @@ -78,38 +102,87 @@ TIMEMORY_PROPERTY_SPECIALIZATION(omnitrace::component::sampling_cpu_clock, OMNITRACE_SAMPLING_CPU_CLOCK, "sampling_cpu_clock", "") TIMEMORY_PROPERTY_SPECIALIZATION(omnitrace::component::sampling_percent, OMNITRACE_SAMPLING_PERCENT, "sampling_percent", "") +TIMEMORY_PROPERTY_SPECIALIZATION(omnitrace::component::sampling_gpu_busy, + OMNITRACE_SAMPLING_GPU_BUSY, "sampling_gpu_busy", + "sampling_gpu_util") +TIMEMORY_PROPERTY_SPECIALIZATION(omnitrace::component::sampling_gpu_memory, + OMNITRACE_SAMPLING_GPU_MEMORY_USAGE, + "sampling_gpu_memory_usage", "") +TIMEMORY_PROPERTY_SPECIALIZATION(omnitrace::component::sampling_gpu_power, + OMNITRACE_SAMPLING_GPU_POWER, "sampling_gpu_power", "") +TIMEMORY_PROPERTY_SPECIALIZATION(omnitrace::component::sampling_gpu_temp, + OMNITRACE_SAMPLING_GPU_TEMP, "sampling_gpu_temp", "") +TIMEMORY_METADATA_SPECIALIZATION(omnitrace::component::roctracer, "roctracer", + "High-precision ROCm API and kernel tracing", "") TIMEMORY_METADATA_SPECIALIZATION(omnitrace::component::sampling_wall_clock, "sampling_wall_clock", "Wall-clock timing", - "derived from statistical sampling") + "Derived from statistical sampling") TIMEMORY_METADATA_SPECIALIZATION(omnitrace::component::sampling_cpu_clock, "sampling_cpu_clock", "CPU-clock timing", - "derived from statistical sampling") + "Derived from statistical sampling") TIMEMORY_METADATA_SPECIALIZATION(omnitrace::component::sampling_percent, "sampling_percent", "Fraction of wall-clock time spent in functions", - "derived from statistical sampling") -TIMEMORY_METADATA_SPECIALIZATION(omnitrace::component::roctracer, "roctracer", - "High-precision ROCm API and kernel tracing", "") + "Derived from statistical sampling") +TIMEMORY_METADATA_SPECIALIZATION(omnitrace::component::sampling_gpu_busy, + "sampling_gpu_busy", + "GPU Utilization (% busy) via ROCm-SMI", + "Derived from sampling") +TIMEMORY_METADATA_SPECIALIZATION(omnitrace::component::sampling_gpu_memory, + "sampling_gpu_memory_usage", + "GPU Memory Usage via ROCm-SMI", "Derived from sampling") +TIMEMORY_METADATA_SPECIALIZATION(omnitrace::component::sampling_gpu_power, + "sampling_gpu_power", "GPU Power Usage via ROCm-SMI", + "Derived from sampling") +TIMEMORY_METADATA_SPECIALIZATION(omnitrace::component::sampling_gpu_temp, + "sampling_gpu_temp", "GPU Temperature via ROCm-SMI", + "Derived from sampling") +// statistics type TIMEMORY_STATISTICS_TYPE(omnitrace::component::sampling_wall_clock, double) TIMEMORY_STATISTICS_TYPE(omnitrace::component::sampling_cpu_clock, double) +TIMEMORY_STATISTICS_TYPE(omnitrace::component::sampling_gpu_busy, double) +TIMEMORY_STATISTICS_TYPE(omnitrace::component::sampling_gpu_temp, double) +TIMEMORY_STATISTICS_TYPE(omnitrace::component::sampling_gpu_power, double) +TIMEMORY_STATISTICS_TYPE(omnitrace::component::sampling_gpu_memory, double) // enable timing units TIMEMORY_DEFINE_CONCRETE_TRAIT(is_timing_category, omnitrace::component::sampling_wall_clock, true_type) -TIMEMORY_DEFINE_CONCRETE_TRAIT(uses_timing_units, - omnitrace::component::sampling_wall_clock, true_type) TIMEMORY_DEFINE_CONCRETE_TRAIT(is_timing_category, omnitrace::component::sampling_cpu_clock, true_type) -TIMEMORY_DEFINE_CONCRETE_TRAIT(uses_timing_units, - omnitrace::component::sampling_cpu_clock, true_type) TIMEMORY_DEFINE_CONCRETE_TRAIT(is_timing_category, omnitrace::component::sampling_percent, true_type) +TIMEMORY_DEFINE_CONCRETE_TRAIT(uses_timing_units, + omnitrace::component::sampling_wall_clock, true_type) +TIMEMORY_DEFINE_CONCRETE_TRAIT(uses_timing_units, + omnitrace::component::sampling_cpu_clock, true_type) +// enable percent units +TIMEMORY_DEFINE_CONCRETE_TRAIT(uses_percent_units, + omnitrace::component::sampling_gpu_busy, true_type) + +// enable memory units +TIMEMORY_DEFINE_CONCRETE_TRAIT(is_memory_category, + omnitrace::component::sampling_gpu_memory, true_type) +TIMEMORY_DEFINE_CONCRETE_TRAIT(uses_memory_units, + omnitrace::component::sampling_gpu_memory, true_type) + +// reporting categories (sum) +TIMEMORY_DEFINE_CONCRETE_TRAIT(report_sum, omnitrace::component::sampling_gpu_busy, + false_type) +TIMEMORY_DEFINE_CONCRETE_TRAIT(report_sum, omnitrace::component::sampling_gpu_temp, + false_type) +TIMEMORY_DEFINE_CONCRETE_TRAIT(report_sum, omnitrace::component::sampling_gpu_power, + false_type) +TIMEMORY_DEFINE_CONCRETE_TRAIT(report_sum, omnitrace::component::sampling_gpu_memory, + false_type) + +// reporting categories (mean) TIMEMORY_DEFINE_CONCRETE_TRAIT(report_mean, omnitrace::component::sampling_percent, false_type) -TIMEMORY_DEFINE_CONCRETE_TRAIT(report_units, omnitrace::component::sampling_percent, - false_type) + +// reporting categories (stats) TIMEMORY_DEFINE_CONCRETE_TRAIT(report_statistics, omnitrace::component::sampling_percent, false_type) diff --git a/projects/rocprofiler-systems/include/library/components/pthread_gotcha.hpp b/projects/rocprofiler-systems/include/library/components/pthread_gotcha.hpp index 7d4c6b6fb5..0824ed5627 100644 --- a/projects/rocprofiler-systems/include/library/components/pthread_gotcha.hpp +++ b/projects/rocprofiler-systems/include/library/components/pthread_gotcha.hpp @@ -56,6 +56,7 @@ struct pthread_gotcha : tim::component::base // generate the gotcha wrappers static void configure(); + static void shutdown(); // threads can set this to avoid starting sampling on child threads static bool& enable_sampling_on_child_threads(); diff --git a/projects/rocprofiler-systems/include/library/components/rocm_smi.hpp b/projects/rocprofiler-systems/include/library/components/rocm_smi.hpp new file mode 100644 index 0000000000..45720cbb62 --- /dev/null +++ b/projects/rocprofiler-systems/include/library/components/rocm_smi.hpp @@ -0,0 +1,165 @@ +// Copyright (c) 2018 Advanced Micro Devices, Inc. All Rights Reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// with the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in the +// documentation and/or other materials provided with the distribution. +// +// * Neither the names of Advanced Micro Devices, Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this Software without specific prior written permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH +// THE SOFTWARE. + +#pragma once + +#include "library/common.hpp" +#include "library/components/fwd.hpp" +#include "library/defines.hpp" +#include "library/thread_data.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace omnitrace +{ +namespace rocm_smi +{ +void +setup(); + +void +shutdown(); + +uint32_t +device_count(); + +struct data +{ + using msec_t = std::chrono::milliseconds; + using usec_t = std::chrono::microseconds; + using nsec_t = std::chrono::nanoseconds; + using promise_t = std::promise; + + using timestamp_t = int64_t; + using power_t = uint64_t; + using busy_perc_t = uint32_t; + using mem_usage_t = uint64_t; + using temp_t = int64_t; + + TIMEMORY_DEFAULT_OBJECT(data) + + explicit data(uint32_t _dev_id); + + void sample(uint32_t _dev_id); + void print(std::ostream& _os) const; + + template , nsec_t>, int> = 0> + static void poll(std::atomic* _state, Tp&& _interval, promise_t*); + + static void set_state(State); + static void poll(std::atomic* _state, nsec_t _interval, promise_t*); + static void post_process(uint32_t _dev_id); + + uint32_t m_dev_id = std::numeric_limits::max(); + timestamp_t m_ts = 0; + busy_perc_t m_busy_perc = 0; + temp_t m_temp = 0; + power_t m_power = 0; + mem_usage_t m_mem_usage = 0; + + friend std::ostream& operator<<(std::ostream& _os, const data& _v) + { + _v.print(_os); + return _os; + } + +private: + friend void omnitrace::rocm_smi::setup(); + friend void omnitrace::rocm_smi::shutdown(); + + static size_t device_count; + static std::set device_list; + static std::unique_ptr polling_finished; + static std::vector& get_initial(); + static std::unique_ptr& get_thread(); + static bool setup(); + static bool shutdown(); +}; + +template < + typename Tp, + std::enable_if_t, std::chrono::nanoseconds>, int>> +void +data::poll(std::atomic* _state, Tp&& _interval, promise_t* _prom) +{ + poll(_state, std::chrono::duration_cast(_interval), _prom); +} + +using bundle_t = std::deque; +using sampler_instances = thread_data; + +#if !defined(OMNITRACE_USE_ROCM_SMI) +inline void +setup() +{} + +inline void +shutdown() +{} + +inline void data::post_process(uint32_t) {} +#endif +} // namespace rocm_smi +} // namespace omnitrace + +#if defined(OMNITRACE_USE_ROCM_SMI) +# if !defined(OMNITRACE_EXTERN_COMPONENTS) || \ + (defined(OMNITRACE_EXTERN_COMPONENTS) && OMNITRACE_EXTERN_COMPONENTS > 0) + +# include +# include +# include + +TIMEMORY_DECLARE_EXTERN_COMPONENT( + TIMEMORY_ESC(data_tracker), true, + double) + +TIMEMORY_DECLARE_EXTERN_COMPONENT( + TIMEMORY_ESC(data_tracker), true, + double) + +TIMEMORY_DECLARE_EXTERN_COMPONENT( + TIMEMORY_ESC(data_tracker), true, + double) + +TIMEMORY_DECLARE_EXTERN_COMPONENT( + TIMEMORY_ESC(data_tracker), true, + double) + +# endif +#endif diff --git a/projects/rocprofiler-systems/include/library/components/roctracer.hpp b/projects/rocprofiler-systems/include/library/components/roctracer.hpp index a197436d9d..5a7f783da1 100644 --- a/projects/rocprofiler-systems/include/library/components/roctracer.hpp +++ b/projects/rocprofiler-systems/include/library/components/roctracer.hpp @@ -52,26 +52,42 @@ struct roctracer static void preinit(); static void global_init() { setup(); } - static void global_finalize() { tear_down(); } + static void global_finalize() { shutdown(); } static bool is_setup(); static void setup(); - static void tear_down(); + static void shutdown(); static void add_setup(const std::string&, std::function&&); - static void add_tear_down(const std::string&, std::function&&); + static void add_shutdown(const std::string&, std::function&&); static void remove_setup(const std::string&); - static void remove_tear_down(const std::string&); + static void remove_shutdown(const std::string&); void start(); void stop(); - void set_prefix(const char* _v) { m_prefix = _v; } - -private: - const char* m_prefix = nullptr; }; + +#if !defined(OMNITRACE_USE_ROCTRACER) +inline void +roctracer::setup() +{} + +inline void +roctracer::shutdown() +{} + +inline bool +roctracer::is_setup() +{ + return false; +} +#endif } // namespace component } // namespace tim +#if !defined(OMNITRACE_USE_ROCTRACER) +TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, component::roctracer_data, false_type) +#endif + TIMEMORY_SET_COMPONENT_API(component::roctracer_data, project::timemory, category::timing, os::supports_unix) TIMEMORY_DEFINE_CONCRETE_TRAIT(is_timing_category, component::roctracer_data, true_type) diff --git a/projects/rocprofiler-systems/include/library/components/roctracer_callbacks.hpp b/projects/rocprofiler-systems/include/library/components/roctracer_callbacks.hpp index 886857b21c..4efc1d4b08 100644 --- a/projects/rocprofiler-systems/include/library/components/roctracer_callbacks.hpp +++ b/projects/rocprofiler-systems/include/library/components/roctracer_callbacks.hpp @@ -90,5 +90,5 @@ roctracer_functions_t& roctracer_setup_routines(); roctracer_functions_t& -roctracer_tear_down_routines(); +roctracer_shutdown_routines(); } // namespace omnitrace diff --git a/projects/rocprofiler-systems/include/library/config.hpp b/projects/rocprofiler-systems/include/library/config.hpp index 60753f264e..d3d2c55302 100644 --- a/projects/rocprofiler-systems/include/library/config.hpp +++ b/projects/rocprofiler-systems/include/library/config.hpp @@ -31,6 +31,7 @@ #include "library/defines.hpp" #include "library/state.hpp" #include "library/timemory.hpp" +#include "timemory/macros/language.hpp" #include @@ -43,10 +44,9 @@ namespace omnitrace // bundle of components around omnitrace_init and omnitrace_finalize using main_bundle_t = tim::lightweight_tuple; + comp::cpu_util, pthread_gotcha_t>; -using gotcha_bundle_t = - tim::lightweight_tuple; +using gotcha_bundle_t = tim::lightweight_tuple; // bundle of components used in instrumentation using instrumentation_bundle_t = @@ -69,26 +69,63 @@ using omnitrace_thread_bundle_t = // // Initialization routines // +inline namespace config +{ void -configure_settings() TIMEMORY_VISIBILITY("default"); +configure_settings(); void -print_config_settings( +print_banner(std::ostream& _os = std::cout); + +void +print_settings( std::ostream& _os, std::function&)>&& _filter); +void +print_settings(); + std::string& get_exe_name(); +template +bool +set_setting_value(const std::string& _name, Tp&& _v) +{ + auto _instance = tim::settings::shared_instance(); + auto _setting = _instance->find(_name); + if(_setting == _instance->end()) return false; + if(!_setting->second) return false; + return _setting->second->set(std::forward(_v)); +} + // // User-configurable settings // std::string get_config_file(); +Mode +get_mode(); + +bool& +is_attached(); + +bool& +is_binary_rewrite(); + +bool +get_is_continuous_integration(); + bool get_debug_env(); +bool +get_debug_init(); + +bool +get_debug_finalize(); + bool get_debug(); @@ -113,6 +150,9 @@ get_use_timemory(); bool& get_use_roctracer(); +bool& +get_use_rocm_smi(); + bool& get_use_sampling(); @@ -125,6 +165,9 @@ get_use_mpip(); bool& get_use_critical_trace(); +bool +get_use_kokkosp(); + bool get_timeline_sampling(); @@ -167,8 +210,9 @@ get_trace_hsa_api_types(); std::string& get_backend(); +// make this visible so omnitrace-avail can call it std::string& -get_perfetto_output_filename(); +get_perfetto_output_filename() TIMEMORY_VISIBILITY("default"); int64_t get_critical_trace_count(); @@ -182,8 +226,15 @@ get_sampling_freq(); double& get_sampling_delay(); +double& +get_rocm_smi_freq(); + +std::string +get_rocm_smi_devices(); + int64_t get_critical_trace_per_row(); +} // namespace config // // Runtime configuration data diff --git a/projects/rocprofiler-systems/include/library/critical_trace.hpp b/projects/rocprofiler-systems/include/library/critical_trace.hpp index 0a1fb2a48f..1886177c0a 100644 --- a/projects/rocprofiler-systems/include/library/critical_trace.hpp +++ b/projects/rocprofiler-systems/include/library/critical_trace.hpp @@ -175,7 +175,7 @@ struct call_chain : private std::vector } template - void generate_perfetto(std::set& _used) const; + void generate_perfetto(std::set& _used, bool _basic = false) const; template bool query(FuncT&&) const; @@ -201,6 +201,11 @@ update(int64_t _tid = threading::get_id()); void compute(int64_t _tid = threading::get_id()); +std::vector> +get_entries( + int64_t _ts, + const std::function& _eval = [](const entry&) { return true; }); + struct id {}; diff --git a/projects/rocprofiler-systems/include/library/debug.hpp b/projects/rocprofiler-systems/include/library/debug.hpp index 0b15d936d4..a671c1333f 100644 --- a/projects/rocprofiler-systems/include/library/debug.hpp +++ b/projects/rocprofiler-systems/include/library/debug.hpp @@ -33,6 +33,8 @@ namespace omnitrace { +inline namespace config +{ bool get_debug(); @@ -44,6 +46,7 @@ get_debug_pid(); bool get_critical_trace_debug(); +} // namespace config } // namespace omnitrace #if defined(TIMEMORY_USE_MPI) @@ -80,8 +83,58 @@ get_critical_trace_debug(); fflush(stderr); \ } +#if defined(TIMEMORY_USE_MPI) +# define OMNITRACE_CONDITIONAL_THROW(COND, ...) \ + if(COND) \ + { \ + char _msg_buffer[2048]; \ + snprintf(_msg_buffer, 2048, "[omnitrace][%i][%li] ", \ + static_cast(tim::dmp::rank()), tim::threading::get_id()); \ + auto len = strlen(_msg_buffer); \ + snprintf(_msg_buffer + len, 2048 - len, __VA_ARGS__); \ + throw std::runtime_error(_msg_buffer); \ + } +#else +# define OMNITRACE_CONDITIONAL_THROW(COND, ...) \ + if(COND) \ + { \ + char _msg_buffer[2048]; \ + snprintf(_msg_buffer, 2048, "[omnitrace][%i][%li] ", \ + static_cast(tim::process::get_id()), \ + tim::threading::get_id()); \ + auto len = strlen(_msg_buffer); \ + snprintf(_msg_buffer + len, 2048 - len, __VA_ARGS__); \ + throw std::runtime_error(_msg_buffer); \ + } +#endif + +#define OMNITRACE_CONDITIONAL_BASIC_THROW(COND, ...) \ + if(COND) \ + { \ + char _msg_buffer[2048]; \ + snprintf(_msg_buffer, 2048, "[omnitrace] " __VA_ARGS__); \ + auto len = strlen(_msg_buffer); \ + snprintf(_msg_buffer + len, 2048 - len, __VA_ARGS__); \ + throw std::runtime_error(_msg_buffer); \ + } + #define OMNITRACE_DEBUG(...) \ OMNITRACE_CONDITIONAL_PRINT(::omnitrace::get_debug(), __VA_ARGS__) #define OMNITRACE_PRINT(...) OMNITRACE_CONDITIONAL_PRINT(true, __VA_ARGS__) #define OMNITRACE_CT_DEBUG(...) \ OMNITRACE_CONDITIONAL_PRINT(::omnitrace::get_critical_trace_debug(), __VA_ARGS__) +#define OMNITRACE_BASIC_PRINT(...) OMNITRACE_CONDITIONAL_BASIC_PRINT(true, __VA_ARGS__) + +#define OMNITRACE_THROW(...) OMNITRACE_CONDITIONAL_THROW(true, __VA_ARGS__) +#define OMNITRACE_BASIC_THROW(...) OMNITRACE_CONDITIONAL_BASIC_THROW(true, __VA_ARGS__) + +#include + +namespace std +{ +inline std::string +to_string(bool _v) +{ + return (_v) ? "true" : "false"; +} +} // namespace std diff --git a/projects/rocprofiler-systems/include/library/defines.hpp.in b/projects/rocprofiler-systems/include/library/defines.hpp.in index c021beff57..488e158f55 100644 --- a/projects/rocprofiler-systems/include/library/defines.hpp.in +++ b/projects/rocprofiler-systems/include/library/defines.hpp.in @@ -36,11 +36,17 @@ // clang-format on #define TIMEMORY_USER_COMPONENT_ENUM \ - OMNITRACE_SAMPLING_WALL_CLOCK_idx, OMNITRACE_SAMPLING_CPU_CLOCK_idx, \ - OMNITRACE_SAMPLING_PERCENT_idx, OMNITRACE_COMPONENT_idx, OMNITRACE_ROCTRACER_idx, + OMNITRACE_COMPONENT_idx, OMNITRACE_ROCTRACER_idx, OMNITRACE_SAMPLING_WALL_CLOCK_idx, \ + OMNITRACE_SAMPLING_CPU_CLOCK_idx, OMNITRACE_SAMPLING_PERCENT_idx, \ + OMNITRACE_SAMPLING_GPU_POWER_idx, OMNITRACE_SAMPLING_GPU_TEMP_idx, \ + OMNITRACE_SAMPLING_GPU_BUSY_idx, OMNITRACE_SAMPLING_GPU_MEMORY_USAGE_idx, -#define OMNITRACE_COMPONENT OMNITRACE_COMPONENT_idx -#define OMNITRACE_ROCTRACER OMNITRACE_ROCTRACER_idx -#define OMNITRACE_SAMPLING_WALL_CLOCK OMNITRACE_SAMPLING_WALL_CLOCK_idx -#define OMNITRACE_SAMPLING_CPU_CLOCK OMNITRACE_SAMPLING_CPU_CLOCK_idx -#define OMNITRACE_SAMPLING_PERCENT OMNITRACE_SAMPLING_PERCENT_idx +#define OMNITRACE_COMPONENT OMNITRACE_COMPONENT_idx +#define OMNITRACE_ROCTRACER OMNITRACE_ROCTRACER_idx +#define OMNITRACE_SAMPLING_WALL_CLOCK OMNITRACE_SAMPLING_WALL_CLOCK_idx +#define OMNITRACE_SAMPLING_CPU_CLOCK OMNITRACE_SAMPLING_CPU_CLOCK_idx +#define OMNITRACE_SAMPLING_PERCENT OMNITRACE_SAMPLING_PERCENT_idx +#define OMNITRACE_SAMPLING_GPU_POWER OMNITRACE_SAMPLING_GPU_POWER_idx +#define OMNITRACE_SAMPLING_GPU_TEMP OMNITRACE_SAMPLING_GPU_TEMP_idx +#define OMNITRACE_SAMPLING_GPU_BUSY OMNITRACE_SAMPLING_GPU_BUSY_idx +#define OMNITRACE_SAMPLING_GPU_MEMORY_USAGE OMNITRACE_SAMPLING_GPU_MEMORY_USAGE_idx diff --git a/projects/rocprofiler-systems/include/library/perfetto.hpp b/projects/rocprofiler-systems/include/library/perfetto.hpp index 090623f69f..9239dd18d8 100644 --- a/projects/rocprofiler-systems/include/library/perfetto.hpp +++ b/projects/rocprofiler-systems/include/library/perfetto.hpp @@ -33,6 +33,9 @@ # define PERFETTO_CATEGORIES \ perfetto::Category("host").SetDescription("Host-side function tracing"), \ perfetto::Category("device").SetDescription("Device-side function tracing"), \ + perfetto::Category("rocm_smi").SetDescription("Device-level metrics"), \ + perfetto::Category("sampling") \ + .SetDescription("Metrics derived from sampling"), \ perfetto::Category("host-critical-trace") \ .SetDescription("Host-side critical traces"), \ perfetto::Category("device-critical-trace") \ @@ -41,6 +44,9 @@ # define PERFETTO_CATEGORIES \ perfetto::Category("host").SetDescription("Host-side function tracing"), \ perfetto::Category("device").SetDescription("Device-side function tracing"), \ + perfetto::Category("rocm_smi").SetDescription("Device-level metrics"), \ + perfetto::Category("sampling") \ + .SetDescription("Metrics derived from sampling"), \ perfetto::Category("host-critical-trace") \ .SetDescription("Host-side critical traces"), \ perfetto::Category("device-critical-trace") \ @@ -87,4 +93,44 @@ public: PERFETTO_DECLARE_DATA_SOURCE_STATIC_MEMBERS(CustomDataSource); #endif + +template +struct perfetto_counter_track +{ + using track_map_t = std::map>; + using name_map_t = std::map>; + using data_t = std::pair; + + static auto init() { (void) get_data(); } + + static auto exists(size_t _idx, int64_t _n = -1) + { + bool _v = get_data().second.count(_idx) != 0; + if(_n < 0 || !_v) return _v; + return static_cast(_n) < get_data().second.at(_idx).size(); + } + + static size_t size(size_t _idx) + { + bool _v = get_data().second.count(_idx) != 0; + if(!_v) return 0; + return get_data().second.at(_idx).size(); + } + + static auto emplace(size_t _idx, const std::string& _v, const char* _units) + { + get_data().first[_idx].emplace_back(_v); + get_data().second[_idx].emplace_back(get_data().first[_idx].back().c_str(), + _units); + } + + static auto& at(size_t _idx, size_t _n) { return get_data().second.at(_idx).at(_n); } + +private: + static data_t& get_data() + { + static auto* _v = new data_t{}; + return *_v; + } +}; } // namespace omnitrace diff --git a/projects/rocprofiler-systems/include/library/redirect.hpp b/projects/rocprofiler-systems/include/library/redirect.hpp new file mode 100644 index 0000000000..6c26cb6d2e --- /dev/null +++ b/projects/rocprofiler-systems/include/library/redirect.hpp @@ -0,0 +1,95 @@ +// MIT License +// +// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#pragma once + +#include +#include +#include +#include +#include + +namespace omnitrace +{ +inline namespace config +{ +bool +get_debug(); + +int +get_verbose(); +} // namespace config + +struct redirect +{ + redirect(std::ostream& _os, std::string _expected) + : m_os{ _os } + , m_expected{ std::move(_expected) } + { + if(!get_debug()) + { + // save stream buffer + m_strm_buffer = m_os.rdbuf(); + // redirect to stringstream + _os.rdbuf(m_buffer.rdbuf()); + } + } + + ~redirect() + { + if(!m_strm_buffer) return; + // restore stream buffer + m_os.rdbuf(m_strm_buffer); + auto _v = m_buffer.str(); + _v = replace(m_buffer.str(), '\n'); + auto _expect = replace(m_expected, '\n'); + if(_v != _expect) + { + if(get_verbose() > 0) + std::cerr << "[omnitrace::redirect] Expected:\n[omnitrace::redirect] " + << _expect + << "\n[omnitrace::redirect] Found:\n[omnitrace::redirect] " + << _v << "\n"; + if(get_verbose() <= 0 || (&m_os != &std::cerr && &m_os != &std::cout)) + m_os << m_buffer.str() << std::flush; + } + } + +private: + template + static std::string replace(std::string _v, Tp _c, const std::string& _s = " ") + { + while(true) + { + auto _pos = _v.find(_c); + if(_pos == std::string::npos) break; + _v = _v.replace(_pos, 1, _s); + } + return _v; + } + + std::ostream& m_os; + std::string m_expected = {}; + std::stringstream m_buffer{}; + std::streambuf* m_strm_buffer = nullptr; +}; +} // namespace omnitrace diff --git a/projects/rocprofiler-systems/include/library/sampling.hpp b/projects/rocprofiler-systems/include/library/sampling.hpp index 59cd75c119..441492f355 100644 --- a/projects/rocprofiler-systems/include/library/sampling.hpp +++ b/projects/rocprofiler-systems/include/library/sampling.hpp @@ -46,6 +46,10 @@ using component::backtrace_cpu_clock; // NOLINT using component::backtrace_fraction; // NOLINT using component::backtrace_wall_clock; // NOLINT using component::sampling_cpu_clock; +using component::sampling_gpu_busy; +using component::sampling_gpu_memory; +using component::sampling_gpu_power; +using component::sampling_gpu_temp; using component::sampling_percent; using component::sampling_wall_clock; @@ -64,7 +68,7 @@ void unblock_signals(std::set = {}); using bundle_t = tim::lightweight_tuple; using sampler_t = tim::sampling::sampler; -using sampler_instances = omnitrace_thread_data; +using sampler_instances = thread_data; std::unique_ptr& get_sampler(int64_t _tid = threading::get_id()); @@ -76,7 +80,7 @@ TIMEMORY_DEFINE_CONCRETE_TRAIT(prevent_reentry, omnitrace::sampling::sampler_t, std::true_type) TIMEMORY_DEFINE_CONCRETE_TRAIT(check_signals, omnitrace::sampling::sampler_t, - std::false_type) + std::true_type) TIMEMORY_DEFINE_CONCRETE_TRAIT(buffer_size, omnitrace::sampling::sampler_t, TIMEMORY_ESC(std::integral_constant)) diff --git a/projects/rocprofiler-systems/include/library/state.hpp b/projects/rocprofiler-systems/include/library/state.hpp index 47ee95ffce..baacf5c7bd 100644 --- a/projects/rocprofiler-systems/include/library/state.hpp +++ b/projects/rocprofiler-systems/include/library/state.hpp @@ -31,7 +31,25 @@ enum class State : unsigned short { DelayedInit = 0, PreInit, + Init, Active, Finalized }; + +enum class Mode : unsigned short +{ + Trace = 0, + Sampling +}; } // namespace omnitrace + +#include + +namespace std +{ +std::string +to_string(omnitrace::State _v); + +std::string +to_string(omnitrace::Mode _v); +} // namespace std diff --git a/projects/rocprofiler-systems/include/library/thread_data.hpp b/projects/rocprofiler-systems/include/library/thread_data.hpp index 327f4b8fa3..ec4b12ba7e 100644 --- a/projects/rocprofiler-systems/include/library/thread_data.hpp +++ b/projects/rocprofiler-systems/include/library/thread_data.hpp @@ -40,7 +40,7 @@ namespace omnitrace static constexpr size_t max_supported_threads = OMNITRACE_MAX_THREADS; template -struct omnitrace_thread_data +struct thread_data { using instance_array_t = std::array, MaxThreads>; using construct_on_init = std::true_type; @@ -58,7 +58,7 @@ struct omnitrace_thread_data template template void -omnitrace_thread_data::construct(Args&&... _args) +thread_data::construct(Args&&... _args) { // construct outside of lambda to prevent data-race static auto& _instances = instances(); @@ -72,14 +72,14 @@ omnitrace_thread_data::construct(Args&&... _args) template std::unique_ptr& -omnitrace_thread_data::instance() +thread_data::instance() { return instances().at(threading::get_id()); } template -typename omnitrace_thread_data::instance_array_t& -omnitrace_thread_data::instances() +typename thread_data::instance_array_t& +thread_data::instances() { static auto _v = instance_array_t{}; return _v; @@ -88,7 +88,7 @@ omnitrace_thread_data::instances() template template std::unique_ptr& -omnitrace_thread_data::instance(construct_on_init, Args&&... _args) +thread_data::instance(construct_on_init, Args&&... _args) { construct(std::forward(_args)...); return instances().at(threading::get_id()); @@ -96,8 +96,8 @@ omnitrace_thread_data::instance(construct_on_init, Args&&.. template template -typename omnitrace_thread_data::instance_array_t& -omnitrace_thread_data::instances(construct_on_init, Args&&... _args) +typename thread_data::instance_array_t& +thread_data::instances(construct_on_init, Args&&... _args) { static auto _v = [&]() { auto _internal = instance_array_t{}; diff --git a/projects/rocprofiler-systems/include/omnitrace.hpp b/projects/rocprofiler-systems/include/omnitrace.hpp index 47e85287e0..a4ca11ee10 100644 --- a/projects/rocprofiler-systems/include/omnitrace.hpp +++ b/projects/rocprofiler-systems/include/omnitrace.hpp @@ -160,6 +160,10 @@ static regexvec_t file_exclude = {}; if(verbose_level >= LEVEL) fprintf(stdout, "[omnitrace][exe] " __VA_ARGS__); \ fflush(stdout); +#define verbprintf_bare(LEVEL, ...) \ + if(verbose_level >= LEVEL) fprintf(stdout, __VA_ARGS__); \ + fflush(stdout); + //======================================================================================// template @@ -477,7 +481,7 @@ dump_info(const string_t& _oname, const fmodset_t& _data, int _level, bool _fail { verbprintf(_level, "Dumping '%s'... ", _oname.c_str()); dump_info(ofs, _data); - verbprintf(_level, "Done\n"); + verbprintf_bare(_level, "Done\n"); } else { @@ -597,7 +601,7 @@ omnitrace_get_address_space(patch_pointer_t& _bpatch, int _cmdc, char** _cmdv, _name.c_str()); throw std::runtime_error("Failed to open binary"); } - verbprintf(1, "Done\n"); + verbprintf_bare(1, "Done\n"); } else if(_pid >= 0) { @@ -611,7 +615,7 @@ omnitrace_get_address_space(patch_pointer_t& _bpatch, int _cmdc, char** _cmdv, (int) _pid); throw std::runtime_error("Failed to attach to process"); } - verbprintf(1, "Done\n"); + verbprintf_bare(1, "Done\n"); } else { @@ -630,7 +634,7 @@ omnitrace_get_address_space(patch_pointer_t& _bpatch, int _cmdc, char** _cmdv, ss.str().c_str()); throw std::runtime_error("Failed to create process"); } - verbprintf(1, "Done\n"); + verbprintf_bare(1, "Done\n"); } return mutatee; diff --git a/projects/rocprofiler-systems/src/avail.cpp b/projects/rocprofiler-systems/src/avail.cpp index c661a773b4..8ce5f2b7fc 100644 --- a/projects/rocprofiler-systems/src/avail.cpp +++ b/projects/rocprofiler-systems/src/avail.cpp @@ -23,14 +23,14 @@ // IN THE SOFTWARE. #include "avail.hpp" - +#include "library/api.hpp" +#include "library/components/backtrace.hpp" #include "library/components/fork_gotcha.hpp" #include "library/components/mpi_gotcha.hpp" #include "library/components/omnitrace.hpp" #include "library/components/pthread_gotcha.hpp" #include "library/components/roctracer.hpp" #include "library/config.hpp" -#include "library/sampling.hpp" #include #include @@ -249,8 +249,6 @@ enum int main(int argc, char** argv) { - omnitrace::configure_settings(); - array_t options = { false, false, false, false, false, false }; array_t fields = {}; array_t use_mark = {}; @@ -436,7 +434,7 @@ main(int argc, char** argv) _parser_set_if_exists(include_hw_counters, "hw-counters"); if(!include_components && !include_settings && !include_hw_counters) - include_components = true; + include_settings = true; if(markdown || include_hw_counters) padding = 6; @@ -457,6 +455,8 @@ main(int argc, char** argv) if(!os) os = &std::cout; + omnitrace_init_library(); + if(include_components) write_component_info(*os, options, use_mark, fields); dump_log(); @@ -699,39 +699,49 @@ write_settings_info(std::ostream& os, const array_t& opts, using width_type = array_t; using width_bool = array_t; - array_type _setting_output; - unique_set _settings_exclude = { "OMNITRACE_ENVIRONMENT", - "OMNITRACE_COMMAND_LINE", - "cereal_class_version", - "settings", - "OMNITRACE_CUDA_EVENT_BATCH_SIZE", - "OMNITRACE_CUPTI_ACTIVITY_KINDS", - "OMNITRACE_CUPTI_ACTIVITY_LEVEL", - "OMNITRACE_CUPTI_DEVICE", - "OMNITRACE_CUPTI_EVENTS", - "OMNITRACE_CUPTI_METRICS", - "OMNITRACE_CUPTI_PCSAMPLING_NUM_COLLECT", - "OMNITRACE_CUPTI_PCSAMPLING_PERIOD", - "OMNITRACE_CUPTI_PCSAMPLING_PER_LINE", - "OMNITRACE_CUPTI_PCSAMPLING_REGION_TOTALS", - "OMNITRACE_CUPTI_PCSAMPLING_SERIALIZED", - "OMNITRACE_CUPTI_PCSAMPLING_STALL_REASONS" }; - - cereal::SettingsTextArchive settings_archive{ _setting_output, _settings_exclude }; - settings::serialize_settings(settings_archive); - width_type _widths = { 0, 0, 0, 0, 0, 0, 0 }; width_bool _wusing = { true, !force_brief, opts[0], opts[1], opts[1], opts[1], opts[2] }; width_bool _mark = { false, false, false, true, true, true, false }; + // this settings has delayed initialization. make sure it is generated + (void) omnitrace::config::get_perfetto_output_filename(); + + array_type _setting_output; + unique_set _settings_exclude = { "OMNITRACE_ENVIRONMENT", "OMNITRACE_COMMAND_LINE", + "cereal_class_version", "settings" }; + +#if !defined(TIMEMORY_USE_CRAYPAT) + _settings_exclude.emplace("OMNITRACE_CRAYPAT"); +#endif + + cereal::SettingsTextArchive settings_archive{ _setting_output, _settings_exclude }; + settings::serialize_settings(settings_archive); + + // exclude some timemory settings which are not relevant to omnitrace + // exact matches, e.g. OMNITRACE_BANNER + std::string _settings_rexclude_exact = + "^OMNITRACE_(BANNER|DESTRUCTOR_REPORT|COMPONENTS|(GLOBAL|MPIP|NCCLP|OMPT|" + "PROFILER|TRACE)_COMPONENTS|PYTHON_EXE|PAPI_ATTACH|PLOT_OUTPUT|SEPARATOR_FREQ|" + "STACK_CLEARING|TARGET_PID|THROTTLE_(COUNT|VALUE)|(AUTO|FLAMEGRAPH)_OUTPUT|" + "(ENABLE|DISABLE)_ALL_SIGNALS|ALLOW_SIGNAL_HANDLER|CTEST_NOTES)$"; + // leading matches, e.g. OMNITRACE_MPI_[A-Z_]+ + std::string _settings_rexclude_begin = + "^OMNITRACE_(ERT|DART|MPI|UPCXX|ROOFLINE|CUDA|NVTX|CUPTI)_[A-Z_]+$"; + + // lambda for deciding which settings we want to restrict displaying + auto&& _remove_conditions = [&_settings_exclude, &_settings_rexclude_exact, + &_settings_rexclude_begin](const auto& itr) { + auto&& _v = itr.find("environ")->second; + bool _a = _settings_exclude.find(_v) != _settings_exclude.end(); + bool _b = std::regex_match(_v, std::regex(_settings_rexclude_exact)); + bool _c = std::regex_match(_v, std::regex(_settings_rexclude_begin)); + return (_a || _b || _c); + }; + _setting_output.erase(std::remove_if(_setting_output.begin(), _setting_output.end(), - [&_settings_exclude](const auto& itr) { - return _settings_exclude.find( - itr.find("environ")->second) != - _settings_exclude.end(); - }), + _remove_conditions), _setting_output.end()); if(alphabetical) diff --git a/projects/rocprofiler-systems/src/library.cpp b/projects/rocprofiler-systems/src/library.cpp index 601b1d31f2..1a5f801b34 100644 --- a/projects/rocprofiler-systems/src/library.cpp +++ b/projects/rocprofiler-systems/src/library.cpp @@ -23,6 +23,7 @@ #include "library.hpp" #include "library/components/fork_gotcha.hpp" #include "library/components/mpi_gotcha.hpp" +#include "library/components/rocm_smi.hpp" #include "library/config.hpp" #include "library/critical_trace.hpp" #include "library/debug.hpp" @@ -31,6 +32,7 @@ #include "library/sampling.hpp" #include "library/thread_data.hpp" #include "library/timemory.hpp" +#include "timemory/mpl/type_traits.hpp" #include #include @@ -58,8 +60,15 @@ ensure_finalization(bool _static_init = false) else { OMNITRACE_CONDITIONAL_PRINT(get_debug_env(), "[%s]\n", __FUNCTION__); + // This environment variable forces the ROCR-Runtime to use polling to wait + // for signals rather than interrupts. We set this variable to avoid issues with + // rocm/roctracer hanging when interrupted by the sampler + // + // see: + // https://github.com/ROCm-Developer-Tools/roctracer/issues/22#issuecomment-572814465 + tim::set_env("HSA_ENABLE_INTERRUPT", "0", 0); } - return scope::destructor{ []() { omnitrace_trace_finalize(); } }; + return scope::destructor{ []() { omnitrace_finalize(); } }; } auto& @@ -105,40 +114,271 @@ using Device = critical_trace::Device; using Phase = critical_trace::Phase; bool -omnitrace_init_tooling() -{ - static bool _once = false; - if(get_state() != State::PreInit || _once) return false; - _once = true; +omnitrace_init_tooling(); +} // namespace +//======================================================================================// +/// +/// \fn void omnitrace_push_trace(const char* name) +/// \brief the "start" function for an instrumentation region +/// +//======================================================================================// + +extern "C" void +omnitrace_push_trace(const char* name) +{ + // return if not active + if(get_state() == State::Finalized) return; + + if(get_state() != State::Active && !omnitrace_init_tooling()) + { + static auto _debug = get_debug_env(); + OMNITRACE_CONDITIONAL_BASIC_PRINT( + _debug, "[%s] %s :: not active and perfetto not initialized\n", __FUNCTION__, + name); + return; + } + + OMNITRACE_DEBUG("[%s] %s\n", __FUNCTION__, name); + + static auto _sample_rate = std::max(get_instrumentation_interval(), 1); + static thread_local size_t _sample_idx = 0; + auto _enabled = (_sample_idx++ % _sample_rate == 0); + get_interval_data().emplace_back(_enabled); + if(_enabled) get_functors().first(name); + if(get_use_critical_trace()) + { + auto _ts = comp::wall_clock::record(); + auto _cid = get_cpu_cid()++; + uint16_t _depth = (get_cpu_cid_stack()->empty()) + ? get_cpu_cid_stack(0)->size() + : get_cpu_cid_stack()->size() - 1; + auto _parent_cid = (get_cpu_cid_stack()->empty()) ? get_cpu_cid_stack(0)->back() + : get_cpu_cid_stack()->back(); + get_cpu_cid_parents().emplace(_cid, std::make_tuple(_parent_cid, _depth)); + add_critical_trace( + threading::get_id(), _cid, 0, _parent_cid, _ts, 0, + critical_trace::add_hash_id(name), _depth); + } +} + +//======================================================================================// +/// +/// \fn void omnitrace_pop_trace(const char* name) +/// \brief the "stop" function for an instrumentation region +/// +//======================================================================================// + +extern "C" void +omnitrace_pop_trace(const char* name) +{ + if(get_state() == State::Active) + { + OMNITRACE_DEBUG("[%s] %s\n", __FUNCTION__, name); + auto& _interval_data = get_interval_data(); + if(!_interval_data.empty()) + { + if(_interval_data.back()) get_functors().second(name); + _interval_data.pop_back(); + } + if(get_use_critical_trace()) + { + if(get_cpu_cid_stack() && !get_cpu_cid_stack()->empty()) + { + auto _cid = get_cpu_cid_stack()->back(); + if(get_cpu_cid_parents().find(_cid) != get_cpu_cid_parents().end()) + { + uint64_t _parent_cid = 0; + uint16_t _depth = 0; + auto _ts = comp::wall_clock::record(); + std::tie(_parent_cid, _depth) = get_cpu_cid_parents().at(_cid); + add_critical_trace( + threading::get_id(), _cid, 0, _parent_cid, _ts, _ts, + critical_trace::add_hash_id(name), _depth); + } + } + } + } + else + { + static auto _debug = get_debug_env(); + OMNITRACE_CONDITIONAL_BASIC_PRINT(_debug, "[%s] %s ignored :: not active\n", + __FUNCTION__, name); + } +} + +//======================================================================================// +/// +/// \fn void omnitrace_set_env(const char* name, const char* env_val) +/// \brief Sets an initial environment variable +/// +//======================================================================================// + +extern "C" void +omnitrace_set_env(const char* env_name, const char* env_val) +{ + // just search env to avoid initializing the settings + OMNITRACE_CONDITIONAL_PRINT(get_debug_env() || get_verbose_env() > 2, + "[%s] Setting env: %s=%s\n", __FUNCTION__, env_name, + env_val); + + tim::set_env(env_name, env_val, 0); + + OMNITRACE_CONDITIONAL_THROW( + get_state() >= State::Init && + (config::get_is_continuous_integration() || get_debug_env()), + "%s(\"%s\", \"%s\") called after omnitrace was initialized. state = %s", + __FUNCTION__, env_name, env_val, std::to_string(get_state()).c_str()); +} + +//======================================================================================// +/// +/// \fn void omnitrace_set_mpi(bool use, bool attached) +/// \brief Configures whether MPI support should be activated +/// +//======================================================================================// + +namespace +{ +bool _set_mpi_called = false; +std::function _start_gotcha_callback = []() {}; +} // namespace + +extern "C" void +omnitrace_set_mpi(bool use, bool attached) +{ + // just search env to avoid initializing the settings + OMNITRACE_CONDITIONAL_PRINT(get_debug_env() || get_verbose_env() > 2, + "[%s] use: %s, attached: %s\n", __FUNCTION__, + (use) ? "y" : "n", (attached) ? "y" : "n"); + + _set_mpi_called = true; + config::is_attached() = attached; + + if(use && !attached && + (get_state() == State::PreInit || get_state() == State::DelayedInit)) + { + tim::set_env("OMNITRACE_USE_PID", "ON", 1); + } + else if(!use) + { + trait::runtime_enabled::set(false); + } + + OMNITRACE_CONDITIONAL_THROW( + get_state() >= State::Init && + (config::get_is_continuous_integration() || get_debug_env()), + "%s(use=%s, attached=%s) called after omnitrace was initialized. state = %s", + __FUNCTION__, std::to_string(use).c_str(), std::to_string(attached).c_str(), + std::to_string(get_state()).c_str()); + + _start_gotcha_callback(); +} + +//======================================================================================// + +extern "C" void +omnitrace_init_library() +{ auto _tid = threading::get_id(); (void) _tid; - auto _mode = tim::get_env("OMNITRACE_MODE", ""); - OMNITRACE_CONDITIONAL_BASIC_PRINT(true, "Instrumentation mode: %s\n", _mode.c_str()); + auto _mode = get_mode(); + + get_state() = State::Init; // configure the settings configure_settings(); - if(gpu::device_count() == 0) + auto _debug_init = get_debug_init(); + auto _debug_value = get_debug(); + if(_debug_init) config::set_setting_value("OMNITRACE_DEBUG", true); + scope::destructor _debug_dtor{ [_debug_value, _debug_init]() { + if(_debug_init) config::set_setting_value("OMNITRACE_DEBUG", _debug_value); + } }; + + OMNITRACE_DEBUG("[%s]\n", __FUNCTION__); + + // below will effectively do: + // get_cpu_cid_stack(0)->emplace_back(-1); + // plus query some env variables + add_critical_trace(0, -1, 0, 0, 0, 0, 0, 0); + + if(gpu::device_count() == 0 && get_state() != State::Active) { - OMNITRACE_DEBUG("No HIP devices were found: disabling roctracer...\n"); + OMNITRACE_DEBUG( + "No HIP devices were found: disabling roctracer and rocm_smi...\n"); get_use_roctracer() = false; + get_use_rocm_smi() = false; } - if(_mode == "sampling") + if(_mode == Mode::Sampling) { - OMNITRACE_PRINT( - "Disabling perfetto, timemory, and critical trace in %s mode...\n", - _mode.c_str()); + OMNITRACE_CONDITIONAL_PRINT(get_verbose() >= 0, + "Disabling critical trace in %s mode...\n", + std::to_string(_mode).c_str()); get_use_sampling() = true; - get_use_timemory() = false; - get_use_perfetto() = false; - get_use_roctracer() = false; get_use_critical_trace() = false; } + tim::trait::runtime_enabled::set(get_use_roctracer()); + tim::trait::runtime_enabled::set(get_use_roctracer()); + + if(get_instrumentation_interval() < 1) get_instrumentation_interval() = 1; + get_interval_data().reserve(512); + + if(get_use_kokkosp()) + { + auto _force = 0; + if(tim::get_env("KOKKOS_PROFILE_LIBRARY") == "libtimemory.so") + _force = 1; + tim::set_env("KOKKOS_PROFILE_LIBRARY", "libomnitrace.so", _force); + } + +#if defined(OMNITRACE_USE_ROCTRACER) + tim::set_env("HSA_TOOLS_LIB", "libomnitrace.so", 0); +#endif +} + +//======================================================================================// + +namespace +{ +bool +omnitrace_init_tooling() +{ + static bool _once = false; + if(get_state() != State::PreInit || get_state() == State::Init || _once) return false; + _once = true; + + OMNITRACE_CONDITIONAL_THROW( + get_state() == State::Init, + "%s called after omnitrace_init_library() was explicitly called", __FUNCTION__); + + OMNITRACE_CONDITIONAL_BASIC_PRINT(get_verbose_env() >= 0, + "Instrumentation mode: %s\n", + std::to_string(config::get_mode()).c_str()); + + if(get_verbose_env() >= 0) print_banner(); + + omnitrace_init_library(); + + OMNITRACE_DEBUG("[%s]\n", __FUNCTION__); + auto _dtor = scope::destructor{ []() { + // if roctracer is not enabled, we cannot rely on OnLoad(...) via HSA tools + // to setup rocm-smi + if constexpr(!trait::is_available::value) + { + try + { + rocm_smi::setup(); + } catch(std::exception& _e) + { + OMNITRACE_PRINT("Exception: %s\n", _e.what()); + } + } + if(get_use_sampling()) { pthread_gotcha::enable_sampling_on_child_threads() = false; @@ -146,6 +386,8 @@ omnitrace_init_tooling() pthread_gotcha::enable_sampling_on_child_threads() = true; sampling::unblock_signals(); } + get_main_bundle()->start(); + get_state()= State::Active; // set to active as very last operation } }; if(get_use_sampling()) @@ -154,9 +396,8 @@ omnitrace_init_tooling() sampling::block_signals(); } - OMNITRACE_DEBUG("[%s]\n", __FUNCTION__); - - if(!get_use_timemory() && !get_use_perfetto() && !get_use_sampling()) + if(!get_use_timemory() && !get_use_perfetto() && !get_use_sampling() && + !get_use_rocm_smi()) { get_state() = State::Finalized; OMNITRACE_DEBUG("[%s] Both perfetto and timemory are disabled. Setting the state " @@ -165,16 +406,6 @@ omnitrace_init_tooling() return false; } - // below will effectively do: - // get_cpu_cid_stack(0)->emplace_back(-1); - // plus query some env variables - add_critical_trace(0, -1, 0, 0, 0, 0, 0, 0); - - tim::trait::runtime_enabled::set(get_use_roctracer()); - - if(get_instrumentation_interval() < 1) get_instrumentation_interval() = 1; - get_interval_data().reserve(512); - if(get_use_timemory()) { comp::user_global_bundle::global_init(); @@ -204,17 +435,6 @@ omnitrace_init_tooling() } } - auto& _main_bundle = get_main_bundle(); - _main_bundle->start(); - -#if defined(OMNITRACE_USE_ROCTRACER) - if(get_use_roctracer()) - { - assert(_main_bundle->get() != nullptr); - assert(_main_bundle->get()->get_is_running()); - } -#endif - perfetto::TracingInitArgs args{}; perfetto::TraceConfig cfg{}; perfetto::protos::gen::TrackEventConfig track_event_cfg{}; @@ -248,17 +468,20 @@ omnitrace_init_tooling() auto _exe = get_exe_name(); static auto _thread_init = [_exe]() { - omnitrace_thread_data::construct( - TIMEMORY_JOIN("", _exe, "/thread-", threading::get_id()), - quirk::config{}); - if(get_use_sampling()) - { - static thread_local auto _once = std::once_flag{}; - std::call_once(_once, sampling::setup); - } + static thread_local auto _thread_setup = [_exe]() { + if(threading::get_id() > 0) + threading::set_thread_name( + TIMEMORY_JOIN(" ", "Thread", threading::get_id()).c_str()); + thread_data::construct( + TIMEMORY_JOIN("", _exe, "/thread-", threading::get_id()), + quirk::config{}); + if(get_use_sampling()) sampling::setup(); + }; + static thread_local auto _once = std::once_flag{}; + std::call_once(_once, _thread_setup); static thread_local auto _dtor = scope::destructor{ []() { if(get_use_sampling()) sampling::shutdown(); - omnitrace_thread_data::instance()->stop(); + thread_data::instance()->stop(); } }; (void) _dtor; }; @@ -318,58 +541,6 @@ omnitrace_init_tooling() get_functors().second = _pop_timemory; } - if(dmp::rank() == 0) - { - static std::set _sample_options = { - "OMNITRACE_SAMPLING_FREQ", "OMNITRACE_SAMPLING_DELAY", - "OMNITRACE_FLAT_SAMPLING", "OMNITRACE_TIMELINE_SAMPLING", - "OMNITRACE_FLAT_SAMPLING", "OMNITRACE_TIMELINE_SAMPLING", - }; - static std::set _perfetto_options = { - "OMNITRACE_OUTPUT_FILE", - "OMNITRACE_BACKEND", - "OMNITRACE_SHMEM_SIZE_HINT_KB", - "OMNITRACE_BUFFER_SIZE_KB", - }; - static std::set _timemory_options = { - "OMNITRACE_ROCTRACER_FLAT_PROFILE", "OMNITRACE_ROCTRACER_TIMELINE_PROFILE" - }; - // generic filter for filtering relevant options - auto _is_omnitrace_option = [](const auto& _v, const auto& _c) { - if(!get_use_roctracer() && _v.find("OMNITRACE_ROCTRACER_") == 0) return false; - if(!get_use_critical_trace() && _v.find("OMNITRACE_CRITICAL_TRACE_") == 0) - return false; - if(!get_use_perfetto() && _perfetto_options.count(_v) > 0) return false; - if(!get_use_timemory() && _timemory_options.count(_v) > 0) return false; - if(!get_use_sampling() && _sample_options.count(_v) > 0) return false; - const auto npos = std::string::npos; - if(_v.find("WIDTH") != npos || _v.find("SEPARATOR_FREQ") != npos || - _v.find("AUTO_OUTPUT") != npos || _v.find("DART_OUTPUT") != npos || - _v.find("FILE_OUTPUT") != npos || _v.find("PLOT_OUTPUT") != npos || - _v.find("FLAMEGRAPH_OUTPUT") != npos) - return false; - if(!_c.empty()) - { - if(_c.find("omnitrace") != _c.end()) return true; - if(_c.find("debugging") != _c.end() && _v.find("DEBUG") != npos) - return true; - if(_c.find("config") != _c.end()) return true; - if(_c.find("dart") != _c.end()) return false; - if(_c.find("io") != _c.end() && _v.find("_OUTPUT") != npos) return true; - if(_c.find("format") != _c.end()) return true; - return false; - } - return (_v.find("OMNITRACE_") == 0); - }; - - if(tim::settings::verbose() > 0) - tim::print_env(std::cerr, [_is_omnitrace_option](const std::string& _v) { - return _is_omnitrace_option(_v, std::set{}); - }); - - print_config_settings(std::cerr, _is_omnitrace_option); - } - if(get_use_perfetto() && !is_system_backend()) { #if defined(CUSTOM_DATA_SOURCE) @@ -392,126 +563,78 @@ omnitrace_init_tooling() tracing_session->StartBlocking(); } - get_state() = State::Active; - // if static objects are destroyed in the inverse order of when they are // created this should ensure that finalization is called before perfetto // ends the tracing session static auto _ensure_finalization = ensure_finalization(); - if(dmp::rank() == 0) puts(""); + if(dmp::rank() == 0 && get_verbose() >= 0) fprintf(stderr, "\n"); return true; } } // namespace -//--------------------------------------------------------------------------------------// +//======================================================================================// extern "C" void -omnitrace_push_trace(const char* name) +omnitrace_init(const char* _mode, bool _is_binary_rewrite, const char* _argv0) { - // return if not active - if(get_state() == State::Finalized) return; + // always the first + std::atexit(&omnitrace_finalize); - if(get_state() != State::Active && !omnitrace_init_tooling()) + OMNITRACE_CONDITIONAL_BASIC_PRINT( + get_debug_env() || get_verbose_env() > 2, + "[%s] mode: %s | is binary rewrite: %s | command: %s\n", __FUNCTION__, _mode, + (_is_binary_rewrite) ? "y" : "n", _argv0); + + tim::set_env("OMNITRACE_MODE", _mode, 0); + config::is_binary_rewrite() = _is_binary_rewrite; + + if(!_set_mpi_called) { - OMNITRACE_DEBUG("[%s] %s :: not active and perfetto not initialized\n", - __FUNCTION__, name); - return; + _start_gotcha_callback = []() { get_gotcha_bundle()->start(); }; } else { - OMNITRACE_DEBUG("[%s] %s\n", __FUNCTION__, name); - } - - static auto _sample_rate = std::max(get_instrumentation_interval(), 1); - static thread_local size_t _sample_idx = 0; - auto _enabled = (_sample_idx++ % _sample_rate == 0); - get_interval_data().emplace_back(_enabled); - if(_enabled) get_functors().first(name); - if(get_use_critical_trace()) - { - auto _ts = comp::wall_clock::record(); - auto _cid = get_cpu_cid()++; - uint16_t _depth = (get_cpu_cid_stack()->empty()) - ? get_cpu_cid_stack(0)->size() - : get_cpu_cid_stack()->size() - 1; - auto _parent_cid = (get_cpu_cid_stack()->empty()) ? get_cpu_cid_stack(0)->back() - : get_cpu_cid_stack()->back(); - get_cpu_cid_parents().emplace(_cid, std::make_tuple(_parent_cid, _depth)); - add_critical_trace( - threading::get_id(), _cid, 0, _parent_cid, _ts, 0, - critical_trace::add_hash_id(name), _depth); + get_gotcha_bundle()->start(); } } -extern "C" void -omnitrace_pop_trace(const char* name) -{ - if(get_state() == State::Active) - { - OMNITRACE_DEBUG("[%s] %s\n", __FUNCTION__, name); - auto& _interval_data = get_interval_data(); - if(!_interval_data.empty()) - { - if(_interval_data.back()) get_functors().second(name); - _interval_data.pop_back(); - } - if(get_use_critical_trace()) - { - if(get_cpu_cid_stack() && !get_cpu_cid_stack()->empty()) - { - auto _cid = get_cpu_cid_stack()->back(); - if(get_cpu_cid_parents().find(_cid) != get_cpu_cid_parents().end()) - { - uint64_t _parent_cid = 0; - uint16_t _depth = 0; - auto _ts = comp::wall_clock::record(); - std::tie(_parent_cid, _depth) = get_cpu_cid_parents().at(_cid); - add_critical_trace( - threading::get_id(), _cid, 0, _parent_cid, _ts, _ts, - critical_trace::add_hash_id(name), _depth); - } - } - } - } - else - { - OMNITRACE_DEBUG("[%s] %s :: not active\n", __FUNCTION__, name); - } -} +//======================================================================================// extern "C" void -omnitrace_trace_init(const char* _info, bool _b, const char* _extra) -{ - OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug_env(), "[%s] %s | %s | %s\n", - __FUNCTION__, _info, (_b) ? "y" : "n", _extra); - auto& _gotcha_bundle = get_gotcha_bundle(); - (void) _gotcha_bundle; -} - -extern "C" void -omnitrace_trace_finalize(void) +omnitrace_finalize(void) { // return if not active if(get_state() != State::Active) return; + pthread_gotcha::enable_sampling_on_child_threads() = false; + + auto _debug_init = get_debug_finalize(); + auto _debug_value = get_debug(); + if(_debug_init) config::set_setting_value("OMNITRACE_DEBUG", true); + scope::destructor _debug_dtor{ [_debug_value, _debug_init]() { + if(_debug_init) config::set_setting_value("OMNITRACE_DEBUG", _debug_value); + } }; + OMNITRACE_DEBUG("[%s]\n", __FUNCTION__); - if(dmp::rank() == 0) puts(""); + auto& _thread_bundle = thread_data::instance(); + if(_thread_bundle) _thread_bundle->stop(); + + if(dmp::rank() == 0 && get_verbose() >= 0) fprintf(stderr, "\n"); + if(get_verbose_env() > 0) config::print_settings(); get_state() = State::Finalized; if(get_use_sampling()) { OMNITRACE_DEBUG("[%s] Shutting down sampling...\n", __FUNCTION__); - pthread_gotcha::enable_sampling_on_child_threads() = false; sampling::shutdown(); sampling::block_signals(); } OMNITRACE_DEBUG("[%s] Stopping gotcha bundle...\n", __FUNCTION__); - // stop the gotcha bundle if(get_gotcha_bundle()) { @@ -519,11 +642,19 @@ omnitrace_trace_finalize(void) get_gotcha_bundle().reset(); } -#if defined(OMNITRACE_USE_ROCTRACER) + pthread_gotcha::shutdown(); + + if(get_use_rocm_smi()) + { + OMNITRACE_DEBUG("[%s] Shutting down rocm-smi sampling...\n", __FUNCTION__); + rocm_smi::shutdown(); + } + OMNITRACE_DEBUG("[%s] Shutting down roctracer...\n", __FUNCTION__); // ensure that threads running roctracer callbacks shutdown - if(get_use_roctracer()) comp::roctracer::tear_down(); -#endif + comp::roctracer::shutdown(); + + if(dmp::rank() == 0) fprintf(stderr, "\n"); OMNITRACE_DEBUG("[%s] Stopping main bundle...\n", __FUNCTION__); // stop the main bundle and report the high-level metrics @@ -551,7 +682,7 @@ omnitrace_trace_finalize(void) // thread-specific data will be wrong if try to stop them from // the main thread. OMNITRACE_DEBUG("[%s] Destroying thread bundle data...\n", __FUNCTION__); - for(auto& itr : omnitrace_thread_data::instances()) + for(auto& itr : thread_data::instances()) { if(itr && itr->get() && !itr->get()->get_is_running()) @@ -559,7 +690,7 @@ omnitrace_trace_finalize(void) std::string _msg = JOIN("", *itr); auto _pos = _msg.find(">>> "); if(_pos != std::string::npos) _msg = _msg.substr(_pos + 5); - OMNITRACE_PRINT("%s\n", _msg.c_str()); + OMNITRACE_CONDITIONAL_PRINT(get_verbose() >= 0, "%s\n", _msg.c_str()); } } @@ -590,7 +721,7 @@ omnitrace_trace_finalize(void) } } - if(get_use_critical_trace()) + if(get_use_critical_trace() || (get_use_rocm_smi() && get_use_roctracer())) { OMNITRACE_DEBUG("[%s] Generating the critical trace...\n", __FUNCTION__); // increase the thread-pool size @@ -600,7 +731,7 @@ omnitrace_trace_finalize(void) for(size_t i = 0; i < max_supported_threads; ++i) { using critical_trace_hash_data = - omnitrace_thread_data; + thread_data; if(critical_trace_hash_data::instances().at(i)) critical_trace::add_hash_id(*critical_trace_hash_data::instances().at(i)); @@ -608,12 +739,34 @@ omnitrace_trace_finalize(void) for(size_t i = 0; i < max_supported_threads; ++i) { - using critical_trace_chain_data = - omnitrace_thread_data; + using critical_trace_chain_data = thread_data; if(critical_trace_chain_data::instances().at(i)) critical_trace::update(i); // launch update task } + } + + // ensure that all the MT instances are flushed + if(get_use_rocm_smi()) + { + size_t _ndevice = gpu::device_count(); + OMNITRACE_CONDITIONAL_PRINT( + get_debug() || get_verbose() > 0, + "[%s] Post-processing rocm-smi samples for %zu devices...\n", __FUNCTION__, + _ndevice); + for(size_t i = 0; i < _ndevice; ++i) + { + rocm_smi::data::post_process(i); + rocm_smi::sampler_instances::instances().at(i).reset(); + } + } + + if(get_use_critical_trace()) + { + OMNITRACE_DEBUG("[%s] Generating the critical trace...\n", __FUNCTION__); + // increase the thread-pool size + tasking::get_critical_trace_thread_pool().initialize_threadpool( + get_critical_trace_num_threads()); // make sure outstanding hash tasks completed before compute OMNITRACE_PRINT("[%s] waiting for all critical trace tasks to complete...\n", @@ -630,7 +783,10 @@ omnitrace_trace_finalize(void) bool _perfetto_output_error = false; if(get_use_perfetto() && !is_system_backend()) { - OMNITRACE_DEBUG("[%s] Flushing perfetto...\n", __FUNCTION__); + if(get_verbose() >= 0) fprintf(stderr, "\n"); + if(get_verbose() >= 0 || get_debug()) + fprintf(stderr, "[%s]|%i> Flushing perfetto...\n", __FUNCTION__, dmp::rank()); + // Make sure the last event is closed for this example. perfetto::TrackEvent::Flush(); @@ -650,14 +806,12 @@ omnitrace_trace_finalize(void) return; } // Write the trace into a file. - fprintf(stderr, - "[%s]> Outputting '%s'. Trace data: %lu B (%.2f KB / %.2f MB / %.2f " - "GB)... ", - __FUNCTION__, get_perfetto_output_filename().c_str(), - (unsigned long) trace_data.size(), - static_cast(trace_data.size()) / units::KB, - static_cast(trace_data.size()) / units::MB, - static_cast(trace_data.size()) / units::GB); + if(get_verbose() >= 0) + fprintf(stderr, "[%s]|%i> Outputting '%s' (%.2f KB / %.2f MB / %.2f GB)... ", + __FUNCTION__, dmp::rank(), get_perfetto_output_filename().c_str(), + static_cast(trace_data.size()) / units::KB, + static_cast(trace_data.size()) / units::MB, + static_cast(trace_data.size()) / units::GB); std::ofstream ofs{}; if(!tim::filepath::open(ofs, get_perfetto_output_filename(), std::ios::out | std::ios::binary)) @@ -669,10 +823,11 @@ omnitrace_trace_finalize(void) else { // Write the trace into a file. - fprintf(stderr, "Done\n"); + if(get_verbose() >= 0) fprintf(stderr, "Done\n"); ofs.write(&trace_data[0], trace_data.size()); } ofs.close(); + if(get_verbose() >= 0) fprintf(stderr, "\n"); } // these should be destroyed before timemory is finalized, especially the @@ -691,33 +846,14 @@ omnitrace_trace_finalize(void) OMNITRACE_DEBUG("[%s] Finalizing timemory... Done\n", __FUNCTION__); if(_perfetto_output_error) - throw std::runtime_error("Unable to create perfetto output file"); -} - -extern "C" void -omnitrace_trace_set_env(const char* env_name, const char* env_val) -{ - // just search env to avoid initializing the settings - OMNITRACE_CONDITIONAL_PRINT(get_debug_env(), "[%s] Setting env: %s=%s\n", - __FUNCTION__, env_name, env_val); - - tim::set_env(env_name, env_val, 0); -} - -extern "C" void -omnitrace_trace_set_mpi(bool use, bool attached) -{ - // just search env to avoid initializing the settings - OMNITRACE_CONDITIONAL_PRINT(get_debug_env(), "[%s] use: %s, attached: %s\n", - __FUNCTION__, (use) ? "y" : "n", (attached) ? "y" : "n"); - if(use && !attached && - (get_state() == State::PreInit || get_state() == State::DelayedInit)) { - tim::set_env("OMNITRACE_USE_PID", "ON", 1); - get_state() = State::DelayedInit; + OMNITRACE_THROW("Error opening perfetto output file: %s", + get_perfetto_output_filename().c_str()); } } +//======================================================================================// + namespace { // if static objects are destroyed randomly (relatively uncommon behavior) diff --git a/projects/rocprofiler-systems/src/library/components/backtrace.cpp b/projects/rocprofiler-systems/src/library/components/backtrace.cpp index e4910589aa..6e01b43915 100644 --- a/projects/rocprofiler-systems/src/library/components/backtrace.cpp +++ b/projects/rocprofiler-systems/src/library/components/backtrace.cpp @@ -21,8 +21,10 @@ // SOFTWARE. #include "library/components/fwd.hpp" +#include "library/components/rocm_smi.hpp" #include "library/config.hpp" #include "library/debug.hpp" +#include "library/perfetto.hpp" #include "library/ptl.hpp" #include "library/sampling.hpp" @@ -44,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -99,10 +102,10 @@ namespace omnitrace namespace component { using hw_counters = typename backtrace::hw_counters; -using signal_type_instances = omnitrace_thread_data, api::sampling>; -using backtrace_init_instances = omnitrace_thread_data; -using sampler_running_instances = omnitrace_thread_data; -using papi_vector_instances = omnitrace_thread_data; +using signal_type_instances = thread_data, api::sampling>; +using backtrace_init_instances = thread_data; +using sampler_running_instances = thread_data; +using papi_vector_instances = thread_data; namespace { @@ -154,9 +157,31 @@ backtrace::preinit() sampling_cpu_clock::label() = "sampling_cpu_clock"; sampling_cpu_clock::description() = "CPU clock time (via sampling)"; - sampling_percent::label() = "sampling_percent"; - sampling_percent::description() = "Percentage of samples"; - sampling_percent::display_unit() = "%"; + sampling_percent::label() = "sampling_percent"; + sampling_percent::description() = "Percentage of samples"; + + sampling_gpu_busy::label() = "sampling_gpu_busy_percent"; + sampling_gpu_busy::description() = "Utilization of GPU(s)"; + sampling_gpu_busy::set_precision(0); + sampling_gpu_busy::set_format_flags(sampling_gpu_busy::get_format_flags() & + std::ios_base::showpoint); + + sampling_gpu_memory::label() = "sampling_gpu_memory_usage"; + sampling_gpu_memory::description() = "Memory usage of GPU(s)"; + + sampling_gpu_power::label() = "sampling_gpu_power"; + sampling_gpu_power::description() = "Power usage of GPU(s)"; + sampling_gpu_power::unit() = units::watt; + sampling_gpu_power::display_unit() = "watts"; + sampling_gpu_power::set_precision(2); + sampling_gpu_power::set_format_flags(sampling_gpu_power::get_format_flags()); + + sampling_gpu_temp::label() = "sampling_gpu_temperature"; + sampling_gpu_temp::description() = "Temperature of GPU(s)"; + sampling_gpu_temp::unit() = 1; + sampling_gpu_temp::display_unit() = "degC"; + sampling_gpu_temp::set_precision(1); + sampling_gpu_temp::set_format_flags(sampling_gpu_temp::get_format_flags()); } std::string @@ -236,6 +261,7 @@ backtrace::sample(int signum) m_tid = threading::get_id(); m_ts = clock_type::now(); m_thr_cpu_ts = tim::get_clock_thread_now(); + m_mem_peak = tim::get_peak_rss(RUSAGE_THREAD); m_data = tim::get_unw_backtrace<128, 4, false>(); auto* itr = m_data.begin(); for(; itr != m_data.end(); ++itr, ++m_size) @@ -295,11 +321,12 @@ backtrace::configure(bool _setup, int64_t _tid) sampling::block_signals(*_signal_types); if constexpr(tim::trait::is_available::value) { + perfetto_counter_track::init(); OMNITRACE_DEBUG("HW COUNTER: starting...\n"); if(get_papi_vector(_tid)) get_papi_vector(_tid)->start(); } - auto _alrm_freq = 1.0 / std::min(get_sampling_freq(), 10.0); + auto _alrm_freq = 1.0 / std::min(get_sampling_freq(), 5.0); auto _prof_freq = 1.0 / get_sampling_freq(); auto _delay = std::max(1.0e-3, get_sampling_delay()); @@ -312,10 +339,14 @@ backtrace::configure(bool _setup, int64_t _tid) _sampler->set_delay(_delay); _sampler->set_frequency(_prof_freq, { SIGPROF }); _sampler->set_frequency(_alrm_freq, { SIGALRM }); + static_assert(tim::trait::buffer_size::value > 0, "Error! Zero buffer size"); - if(_sampler->get_buffer_size() == 0) - throw std::runtime_error("dynamic sampler has a zero buffer size"); + + OMNITRACE_CONDITIONAL_THROW( + _sampler->get_buffer_size() <= 0, + "dynamic sampler requires a positive buffer size: %zu", + _sampler->get_buffer_size()); OMNITRACE_DEBUG("Sampler for thread %lu will be triggered %5.1fx per second " "(every %5.2e seconds)...\n", @@ -365,6 +396,8 @@ backtrace::get_last_hwcounters() void backtrace::post_process(int64_t _tid) { + namespace quirk = tim::quirk; + configure(false, _tid); auto& _sampler = sampling::sampler_instances::instances().at(_tid); @@ -392,9 +425,12 @@ backtrace::post_process(int64_t _tid) // debugging feature static bool _keep_internal = tim::get_env("OMNITRACE_SAMPLING_KEEP_INTERNAL", get_debug()); - if(_keep_internal) return 1; const auto _npos = std::string::npos; + if(_keep_internal) return 1; if(_lbl.find("omnitrace_init_tooling") != _npos) return -1; + if(_lbl.find("omnitrace_push_trace") != _npos) return -1; + if(_lbl.find("omnitrace_pop_trace") != _npos) return -1; + if(_lbl.find("amd_comgr_") == 0) return -1; if(_check_internal) { if(std::regex_search( @@ -423,11 +459,141 @@ backtrace::post_process(int64_t _tid) return _lbl.replace(_pos, _dyninst.length(), ""); }; - auto _data = _sampler->get_allocator().get_data(); + using common_type_t = typename hw_counters::common_type; + auto _hw_cnt_labels = (get_papi_vector(_tid)) + ? comp::papi_common::get_events() + : std::vector{}; + + auto _process_perfetto_counters = [&](const std::vector& _data) { + if(!perfetto_counter_track::exists(_tid)) + { + auto _thrname = TIMEMORY_JOIN("", "[Thread ", _tid, "] "); + auto addendum = [&](const std::string& _v) { return _thrname + _v + " (S)"; }; + perfetto_counter_track::emplace( + _tid, addendum("Peak Memory Usage"), "MB"); + } + + if(!perfetto_counter_track::exists(_tid) && + tim::trait::runtime_enabled::get()) + { + auto _thrname = TIMEMORY_JOIN("", "[Thread ", _tid, "] "); + auto addendum = [&](const std::string& _v) { return _thrname + _v + " (S)"; }; + for(auto& itr : _hw_cnt_labels) + { + perfetto_counter_track::emplace( + _tid, addendum(tim::papi::get_event_info(itr).short_descr), ""); + } + } + + for(const auto& ditr : _data) + { + const auto* _bt = ditr->get(); + if(_bt->m_tid != _tid) continue; + + auto _ts = static_cast(_bt->m_ts.time_since_epoch().count()); + + TRACE_COUNTER("sampling", perfetto_counter_track::at(_tid, 0), + _ts, _bt->m_mem_peak / units::megabyte); + + if(tim::trait::runtime_enabled::get()) + { + for(size_t i = 0; i < perfetto_counter_track::size(_tid); + ++i) + { + if(i < _bt->m_hw_counter.size()) + { + TRACE_COUNTER("sampling", + perfetto_counter_track::at(_tid, i), + _ts, _bt->m_hw_counter.at(i)); + } + } + } + } + }; + + auto _process_perfetto = [&](const std::vector& _data, + bool _rename) { + if(_rename) + threading::set_thread_name(TIMEMORY_JOIN(" ", "Thread", _tid, "(S)").c_str()); + time_point_type _last_wall_ts = _init->get_timestamp(); + + for(const auto& ditr : _data) + { + const auto* _bt = ditr->get(); + if(_bt->m_tid != _tid) continue; + + static std::set _static_strings{}; + std::string _last = {}; + for(const auto& itr : _bt->get()) + { + auto _name = tim::demangle(_patch_label(itr)); + auto _use = + _use_label(_name, !_last.empty() && + (_last == "start_thread" || _last == "clone")); + if(_use == -1) break; + if(_use == 0) continue; + auto sitr = _static_strings.emplace(_name); + _last = *sitr.first; + auto _ts = static_cast(_bt->m_ts.time_since_epoch().count()); + + TRACE_EVENT_BEGIN( + "sampling", perfetto::StaticString{ sitr.first->c_str() }, + static_cast(_last_wall_ts.time_since_epoch().count())); + TRACE_EVENT_END("sampling", _ts); + } + _last_wall_ts = _bt->m_ts; + } + }; + + auto _raw_data = _sampler->get_allocator().get_data(); // single sample that is useless (backtrace to unblocking signals) - if(_data.size() == 1 && _data.front().size() <= 1) _data.clear(); - OMNITRACE_DEBUG("Post-processing %zu sampling entries for thread %lu...\n", - _data.size(), _tid); + if(_raw_data.size() == 1 && _raw_data.front().size() <= 1) _raw_data.clear(); + + std::vector _data{}; + for(auto& ditr : _raw_data) + { + _data.reserve(_data.size() + ditr.size()); + for(auto& ritr : ditr) + { + auto* _bt = ritr.get(); + if(!_bt) + { + OMNITRACE_PRINT( + "Warning! Nullptr to backtrace instance for thread %lu...\n", _tid); + continue; + } + if(_bt->empty()) continue; + _data.emplace_back(&ritr); + } + } + + if(_data.empty()) return; + + OMNITRACE_CONDITIONAL_PRINT( + get_verbose() >= 0 || get_debug(), + "Post-processing %zu sampling entries for thread %lu...\n", _data.size(), _tid); + + std::sort(_data.begin(), _data.end(), + [](const sampling::bundle_t* _lhs, const sampling::bundle_t* _rhs) { + return _lhs->get()->m_ts < _rhs->get()->m_ts; + }); + + if(get_use_perfetto()) + { + _process_perfetto_counters(_data); + + if(_tid == 0 && get_mode() == Mode::Sampling) + _process_perfetto(_data, false); + else + { + auto _v = pthread_gotcha::enable_sampling_on_child_threads(); + pthread_gotcha::enable_sampling_on_child_threads() = false; + std::thread{ _process_perfetto, _data, true }.join(); + pthread_gotcha::enable_sampling_on_child_threads() = _v; + } + } + + if(!get_use_timemory()) return; std::map> _depth_sum = {}; auto _scope = tim::scope::config{}; @@ -441,125 +607,101 @@ backtrace::post_process(int64_t _tid) using bundle_t = tim::lightweight_tuple; - for(auto& ritr : ditr) + auto* _bt = ditr->get(); + + double _elapsed_wc = (_bt->m_ts - _last_wall_ts).count(); + double _elapsed_cc = (_bt->m_thr_cpu_ts - _last_cpu_ts); + + std::vector _tc{}; + _tc.reserve(_bt->size()); + + // generate the instances of the tuple of components and start them + for(const auto& itr : _bt->get()) { - auto* _bt = ritr.get(); - if(!_bt) - { - OMNITRACE_PRINT( - "Warning! Nullptr to backtrace instance for thread %lu...\n", _tid); - continue; - } - - if(_bt->empty()) continue; - - double _elapsed_wc = (_bt->m_ts - _last_wall_ts).count(); - double _elapsed_cc = (_bt->m_thr_cpu_ts - _last_cpu_ts); - - std::vector _tc{}; - _tc.reserve(_bt->size()); - - // generate the instances of the tuple of components and start them - for(const auto& itr : _bt->get()) - { - auto _lbl = _patch_label(itr); - auto _use = _use_label(_lbl, !_tc.empty() && - (_tc.back().key() == "start_thread" || + auto _lbl = _patch_label(itr); + auto _use = + _use_label(_lbl, !_tc.empty() && (_tc.back().key() == "start_thread" || _tc.back().key() == "clone")); - if(_use == -1) break; - if(_use == 0) continue; - _tc.emplace_back(tim::string_view_t{ _lbl }, _scope); - _tc.back().push(_bt->m_tid); - _tc.back().start(); - } - - // stop the instances and update the values as needed - for(size_t i = 0; i < _tc.size(); ++i) - { - auto& itr = _tc.at(_tc.size() - i - 1); - size_t _depth = 0; - _depth_sum[_bt->m_tid][_depth] += 1; - itr.stop(); - if constexpr(tim::trait::is_available::value) - { - auto* _sc = itr.get(); - if(_sc) - { - auto _value = _elapsed_wc / sampling_wall_clock::get_unit(); - _sc->set_value(_value); - _sc->set_accum(_value); - } - } - if constexpr(tim::trait::is_available::value) - { - auto* _cc = itr.get(); - if(_cc) - { - _cc->set_value(_elapsed_cc / sampling_cpu_clock::get_unit()); - _cc->set_accum(_elapsed_cc / sampling_cpu_clock::get_unit()); - } - } - if constexpr(tim::trait::is_available::value) - { - auto* _hw_counter = itr.get(); - if(_hw_counter) - { - _hw_counter->set_value(_bt->m_hw_counter); - _hw_counter->set_accum(_bt->m_hw_counter); - } - } - itr.pop(); - } - _last_wall_ts = _bt->m_ts; - _last_cpu_ts = _bt->m_thr_cpu_ts; + if(_use == -1) break; + if(_use == 0) continue; + _tc.emplace_back(tim::string_view_t{ _lbl }, _scope); + _tc.back().push(_bt->m_tid); + _tc.back().start(); } - } - namespace quirk = tim::quirk; + // stop the instances and update the values as needed + for(size_t i = 0; i < _tc.size(); ++i) + { + auto& itr = _tc.at(_tc.size() - i - 1); + size_t _depth = 0; + _depth_sum[_bt->m_tid][_depth] += 1; + itr.stop(); + if constexpr(tim::trait::is_available::value) + { + auto* _sc = itr.get(); + if(_sc) + { + auto _value = _elapsed_wc / sampling_wall_clock::get_unit(); + _sc->set_value(_value); + _sc->set_accum(_value); + } + } + if constexpr(tim::trait::is_available::value) + { + auto* _cc = itr.get(); + if(_cc) + { + _cc->set_value(_elapsed_cc / sampling_cpu_clock::get_unit()); + _cc->set_accum(_elapsed_cc / sampling_cpu_clock::get_unit()); + } + } + if constexpr(tim::trait::is_available::value) + { + auto* _hw_counter = itr.get(); + if(_hw_counter) + { + _hw_counter->set_value(_bt->m_hw_counter); + _hw_counter->set_accum(_bt->m_hw_counter); + } + } + itr.pop(); + } + _last_wall_ts = _bt->m_ts; + _last_cpu_ts = _bt->m_thr_cpu_ts; + } for(auto&& ditr : _data) { using bundle_t = tim::lightweight_tuple>; - for(auto& ritr : ditr) + auto* _bt = ditr->get(); + + std::vector _tc{}; + _tc.reserve(_bt->size()); + + // generate the instances of the tuple of components and start them + for(const auto& itr : _bt->get()) { - auto* _bt = ritr.get(); - if(!_bt) - { - OMNITRACE_PRINT( - "Warning! Nullptr to backtrace instance for thread %lu...\n", _tid); - continue; - } + auto _lbl = _patch_label(itr); + auto _use = + _use_label(_lbl, !_tc.empty() && _tc.back().key() == "start_thread"); + if(_use == -1) break; + if(_use == 0) continue; + _tc.emplace_back(tim::string_view_t{ _lbl }); + _tc.back().push(_bt->m_tid); + _tc.back().start(); + } - if(_bt->empty()) continue; - - std::vector _tc{}; - _tc.reserve(_bt->size()); - - // generate the instances of the tuple of components and start them - for(const auto& itr : _bt->get()) - { - auto _lbl = _patch_label(itr); - auto _use = - _use_label(_lbl, !_tc.empty() && _tc.back().key() == "start_thread"); - if(_use == -1) break; - if(_use == 0) continue; - _tc.emplace_back(tim::string_view_t{ _lbl }); - _tc.back().push(_bt->m_tid); - _tc.back().start(); - } - - // stop the instances and update the values as needed - for(size_t i = 0; i < _tc.size(); ++i) - { - auto& itr = _tc.at(_tc.size() - i - 1); - size_t _depth = 0; - double _value = (1.0 / _depth_sum[_bt->m_tid][_depth]) * 100.0; - itr.store(std::plus{}, _value); - itr.stop(); - itr.pop(); - } + // stop the instances and update the values as needed + for(size_t i = 0; i < _tc.size(); ++i) + { + auto& itr = _tc.at(_tc.size() - i - 1); + size_t _depth = 0; + double _value = (1.0 / _depth_sum[_bt->m_tid][_depth]) * 100.0; + itr.store(std::plus{}, _value); + itr.stop(); + itr.pop(); } } } diff --git a/projects/rocprofiler-systems/src/library/components/mpi_gotcha.cpp b/projects/rocprofiler-systems/src/library/components/mpi_gotcha.cpp index 16f9062169..be19210145 100644 --- a/projects/rocprofiler-systems/src/library/components/mpi_gotcha.cpp +++ b/projects/rocprofiler-systems/src/library/components/mpi_gotcha.cpp @@ -21,9 +21,11 @@ // SOFTWARE. #include "library/components/mpi_gotcha.hpp" +#include "library/api.hpp" #include "library/components/omnitrace.hpp" #include "library/config.hpp" #include "library/debug.hpp" +#include "timemory/backends/process.hpp" #include #include @@ -35,7 +37,7 @@ namespace uint64_t mpip_index = std::numeric_limits::max(); std::string mpi_init_string = {}; -// this ensures omnitrace_trace_finalize is called before MPI_Finalize +// this ensures omnitrace_finalize is called before MPI_Finalize void omnitrace_mpi_set_attr() { @@ -49,9 +51,7 @@ omnitrace_mpi_set_attr() if(mpip_index != std::numeric_limits::max()) comp::deactivate_mpip, api::omnitrace>(mpip_index); - if(!mpi_init_string.empty()) omnitrace_pop_trace(mpi_init_string.c_str()); - mpi_init_string = {}; - omnitrace_trace_finalize(); + omnitrace_finalize(); return MPI_SUCCESS; }; using copy_func_t = int (*)(MPI_Comm, int, void*, void*, void*, int*); @@ -73,8 +73,10 @@ mpi_gotcha::configure() mpi_gotcha_t::template configure<1, int, int*, char***, int, int*>( "MPI_Init_thread"); mpi_gotcha_t::template configure<2, int>("MPI_Finalize"); - mpi_gotcha_t::template configure<3, int, tim::mpi::comm_t, int*>("MPI_Comm_rank"); - mpi_gotcha_t::template configure<4, int, tim::mpi::comm_t, int*>("MPI_Comm_size"); +#if defined(OMNITRACE_USE_MPI_HEADERS) + mpi_gotcha_t::template configure<3, int, comm_t, int*>("MPI_Comm_rank"); + mpi_gotcha_t::template configure<4, int, comm_t, int*>("MPI_Comm_size"); +#endif }; } @@ -83,15 +85,15 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming, int*, char***) { OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug_env(), "[%s] %s(int*, char***)\n", __FUNCTION__, _data.tool_id.c_str()); + if(get_state() == ::omnitrace::State::DelayedInit) - { - get_state() = ::omnitrace::State::PreInit; - mpi_init_string = _data.tool_id; + get_state() = ::omnitrace::State::PreInit; + + omnitrace_push_trace(_data.tool_id.c_str()); #if !defined(TIMEMORY_USE_MPI) && defined(TIMEMORY_USE_MPI_HEADERS) - tim::mpi::is_initialized_callback() = []() { return true; }; - tim::mpi::is_finalized() = false; + tim::mpi::is_initialized_callback() = []() { return true; }; + tim::mpi::is_finalized() = false; #endif - } } void @@ -101,14 +103,13 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming, int*, char***, in "[%s] %s(int*, char***, int, int*)\n", __FUNCTION__, _data.tool_id.c_str()); if(get_state() == ::omnitrace::State::DelayedInit) - { - get_state() = ::omnitrace::State::PreInit; - mpi_init_string = _data.tool_id; + get_state() = ::omnitrace::State::PreInit; + + omnitrace_push_trace(_data.tool_id.c_str()); #if !defined(TIMEMORY_USE_MPI) && defined(TIMEMORY_USE_MPI_HEADERS) - tim::mpi::is_initialized_callback() = []() { return true; }; - tim::mpi::is_finalized() = false; + tim::mpi::is_initialized_callback() = []() { return true; }; + tim::mpi::is_finalized() = false; #endif - } } void @@ -116,16 +117,16 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming) { OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug_env(), "[%s] %s()\n", __FUNCTION__, _data.tool_id.c_str()); + if(mpip_index != std::numeric_limits::max()) comp::deactivate_mpip, api::omnitrace>(mpip_index); - if(!mpi_init_string.empty()) omnitrace_pop_trace(mpi_init_string.c_str()); - mpi_init_string = {}; + #if !defined(TIMEMORY_USE_MPI) && defined(TIMEMORY_USE_MPI_HEADERS) tim::mpi::is_initialized_callback() = []() { return false; }; tim::mpi::is_finalized() = true; #else - omnitrace_trace_finalize(); + omnitrace_finalize(); #endif } @@ -134,6 +135,8 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming, comm_t, int* _val { OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug_env(), "[%s] %s()\n", __FUNCTION__, _data.tool_id.c_str()); + + omnitrace_push_trace(_data.tool_id.c_str()); if(_data.tool_id == "MPI_Comm_rank") { m_rank_ptr = _val; @@ -144,8 +147,8 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming, comm_t, int* _val } else { - OMNITRACE_PRINT("[%s] %s(, %p) :: unexpected function wrapper\n", - __FUNCTION__, _data.tool_id.c_str(), _val); + OMNITRACE_BASIC_PRINT("[%s] %s(, %p) :: unexpected function wrapper\n", + __FUNCTION__, _data.tool_id.c_str(), _val); } } @@ -154,8 +157,8 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::outgoing, int _retval) { OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug_env(), "[%s] %s() returned %i\n", __FUNCTION__, _data.tool_id.c_str(), (int) _retval); - if(_retval == tim::mpi::success_v && get_state() == ::omnitrace::State::PreInit && - _data.tool_id.find("MPI_Init") == 0) + + if(_retval == tim::mpi::success_v && _data.tool_id.find("MPI_Init") == 0) { omnitrace_mpi_set_attr(); // omnitrace will set this environement variable to true in binary rewrite mode @@ -165,8 +168,9 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::outgoing, int _retval) // were excluded via a regex expression) if(get_use_mpip()) { - OMNITRACE_CONDITIONAL_BASIC_PRINT( - get_debug_env(), "[%s] Activating MPI wrappers...\n", __FUNCTION__); + OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug_env() || get_verbose_env() > 0, + "[%s] Activating MPI wrappers...\n", + __FUNCTION__); // use env vars OMNITRACE_MPIP_PERMIT_LIST and OMNITRACE_MPIP_REJECT_LIST // to control the gotcha bindings at runtime @@ -176,7 +180,6 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::outgoing, int _retval) comp::activate_mpip, api::omnitrace>(); } - omnitrace_push_trace(_data.tool_id.c_str()); } else if(_retval == tim::mpi::success_v && _data.tool_id.find("MPI_Comm_") == 0) { @@ -185,17 +188,17 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::outgoing, int _retval) if(m_rank_ptr) { m_rank = std::max(*m_rank_ptr, m_rank); - OMNITRACE_CONDITIONAL_BASIC_PRINT(tim::settings::verbose() > 0, - "MPI rank: %i\n", m_rank); tim::mpi::set_rank(m_rank); tim::settings::default_process_suffix() = m_rank; get_perfetto_output_filename().clear(); - (void) get_perfetto_output_filename(); + OMNITRACE_CONDITIONAL_BASIC_PRINT( + get_debug() || get_verbose() > 0, "[pid=%i] MPI rank: %i (%i)\n", + process::get_id(), tim::mpi::rank(), m_rank); } else { - OMNITRACE_PRINT("[%s] %s() returned %i :: nullptr to rank\n", - __FUNCTION__, _data.tool_id.c_str(), (int) _retval); + OMNITRACE_BASIC_PRINT("[%s] %s() returned %i :: nullptr to rank\n", + __FUNCTION__, _data.tool_id.c_str(), (int) _retval); } } else if(_data.tool_id == "MPI_Comm_size") @@ -203,22 +206,25 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::outgoing, int _retval) if(m_size_ptr) { m_size = std::max(*m_size_ptr, m_size); - OMNITRACE_CONDITIONAL_BASIC_PRINT(tim::settings::verbose() > 0, - "MPI size: %i\n", m_size); tim::mpi::set_size(m_size); + OMNITRACE_CONDITIONAL_BASIC_PRINT( + get_debug() || get_verbose() > 0, "[pid=%i] MPI size: %i (%i)\n", + process::get_id(), tim::mpi::size(), m_size); } else { - OMNITRACE_PRINT("[%s] %s() returned %i :: nullptr to size\n", - __FUNCTION__, _data.tool_id.c_str(), (int) _retval); + OMNITRACE_BASIC_PRINT("[%s] %s() returned %i :: nullptr to size\n", + __FUNCTION__, _data.tool_id.c_str(), (int) _retval); } } else { - OMNITRACE_PRINT("[%s] %s() returned %i :: unexpected function wrapper\n", - __FUNCTION__, _data.tool_id.c_str(), (int) _retval); + OMNITRACE_BASIC_PRINT( + "[%s] %s() returned %i :: unexpected function wrapper\n", __FUNCTION__, + _data.tool_id.c_str(), (int) _retval); } } + omnitrace_pop_trace(_data.tool_id.c_str()); } } // namespace omnitrace diff --git a/projects/rocprofiler-systems/src/library/components/pthread_gotcha.cpp b/projects/rocprofiler-systems/src/library/components/pthread_gotcha.cpp index dcf24dd067..94a3fd59c8 100644 --- a/projects/rocprofiler-systems/src/library/components/pthread_gotcha.cpp +++ b/projects/rocprofiler-systems/src/library/components/pthread_gotcha.cpp @@ -21,10 +21,13 @@ // SOFTWARE. #include "library/components/pthread_gotcha.hpp" +#include "library/components/omnitrace.hpp" +#include "library/components/roctracer.hpp" #include "library/config.hpp" #include "library/debug.hpp" #include "library/sampling.hpp" +#include #include #include @@ -40,6 +43,47 @@ std::set shutdown(); } // namespace sampling +namespace mpl = tim::mpl; + +using bundle_t = tim::lightweight_tuple; +using wall_pw_t = mpl::piecewise_select; // only wall-clock +using main_pw_t = mpl::piecewise_ignore; // exclude wall-clock +using omni_pw_t = mpl::piecewise_select<>; + +namespace +{ +std::map> bundles = {}; +std::mutex bundles_mutex{}; + +inline void +start_bundle(bundle_t& _bundle) +{ + if(comp::roctracer::is_setup()) + { + _bundle.push(main_pw_t{}); + _bundle.start(); + } + else + { + _bundle.push(omni_pw_t{}); + _bundle.start(omni_pw_t{}); + } +} + +inline void +stop_bundle(bundle_t& _bundle, int64_t _tid) +{ + _bundle.stop(wall_pw_t{}); // stop wall-clock so we can get the value + // update roctracer_data + _bundle.store(std::plus{}, + _bundle.get()->get() * units::sec); + // stop all other components including roctracer_data after update + _bundle.stop(main_pw_t{}); + // exclude popping wall-clock + _bundle.pop(main_pw_t{}, _tid); +} +} // namespace + pthread_gotcha::wrapper::wrapper(routine_t _routine, void* _arg, bool _enable_sampling, promise_t* _p) : m_enable_sampling{ _enable_sampling } @@ -51,10 +95,41 @@ pthread_gotcha::wrapper::wrapper(routine_t _routine, void* _arg, bool _enable_sa void* pthread_gotcha::wrapper::operator()() const { - std::set _signals{}; - auto& _enable_sampling = pthread_gotcha::enable_sampling_on_child_threads(); - if(m_enable_sampling && _enable_sampling) + std::shared_ptr _bundle{}; + std::set _signals{}; + auto& _enable_sampling = pthread_gotcha::enable_sampling_on_child_threads(); + auto _active = (get_state() == omnitrace::State::Active); + int64_t _tid = -1; + auto _is_sampling = false; + auto _dtor = scope::destructor{ [&]() { + if(_is_sampling) + { + sampling::block_signals(_signals); + sampling::shutdown(); + } + + if(_bundle) + { + std::unique_lock _lk{ bundles_mutex }; + stop_bundle(*_bundle, _tid); + _bundle.reset(); + bundles.erase(_tid); + } + } }; + + if(m_enable_sampling && _enable_sampling && _active) { + _tid = threading::get_id(); + threading::set_thread_name(TIMEMORY_JOIN(" ", "Thread", _tid).c_str()); + { + std::unique_lock _lk{ bundles_mutex }; + if(comp::roctracer::is_setup()) + _bundle = + bundles.emplace(_tid, std::make_shared("start_thread")) + .first->second; + } + if(_bundle) start_bundle(*_bundle); + _is_sampling = true; _enable_sampling = false; _signals = sampling::setup(); _enable_sampling = true; @@ -64,15 +139,7 @@ pthread_gotcha::wrapper::operator()() const if(m_promise) m_promise->set_value(); // execute the original function - auto* _ret = m_routine(m_arg); - - if(m_enable_sampling && _enable_sampling) - { - sampling::block_signals(_signals); - sampling::shutdown(); - } - - return _ret; + return m_routine(m_arg); } void* @@ -95,6 +162,27 @@ pthread_gotcha::configure() }; } +void +pthread_gotcha::shutdown() +{ + std::unique_lock _lk{ bundles_mutex }; + unsigned long _ndangling = 0; + for(auto itr : bundles) + { + if(itr.second) + { + stop_bundle(*itr.second, itr.first); + ++_ndangling; + } + itr.second.reset(); + } + + OMNITRACE_CONDITIONAL_PRINT( + (get_verbose() > 0 || get_debug()) && _ndangling > 0, + "pthread_gotcha::shutdown() cleaned up %lu dangling bundles\n", _ndangling); + bundles.clear(); +} + bool& pthread_gotcha::enable_sampling_on_child_threads() { @@ -107,13 +195,18 @@ int pthread_gotcha::operator()(pthread_t* thread, const pthread_attr_t* attr, void* (*start_routine)(void*), void* arg) const { - auto _enable_sampling = enable_sampling_on_child_threads(); + bundle_t _bundle{ "pthread_create" }; + auto _enable_sampling = enable_sampling_on_child_threads(); - if(!_enable_sampling) + if(!get_use_sampling() || !_enable_sampling) { + // if(!get_use_sampling()) start_bundle(_bundle); auto* _obj = new wrapper(start_routine, arg, _enable_sampling, nullptr); // create the thread - return pthread_create(thread, attr, &wrapper::wrap, static_cast(_obj)); + auto _ret = + pthread_create(thread, attr, &wrapper::wrap, static_cast(_obj)); + // if(!get_use_sampling()) stop_bundle(_bundle, threading::get_id()); + return _ret; } // block the signals in entire process @@ -121,18 +214,22 @@ pthread_gotcha::operator()(pthread_t* thread, const pthread_attr_t* attr, tim::sampling::block_signals({ SIGALRM, SIGPROF }, tim::sampling::sigmask_scope::process); + start_bundle(_bundle); + // promise set by thread when signal handler is configured auto _promise = std::promise{}; auto _fut = _promise.get_future(); - auto* _obj = new wrapper(start_routine, arg, _enable_sampling, &_promise); + auto* _wrap = new wrapper(start_routine, arg, _enable_sampling, &_promise); // create the thread - auto _ret = pthread_create(thread, attr, &wrapper::wrap, static_cast(_obj)); + auto _ret = pthread_create(thread, attr, &wrapper::wrap, static_cast(_wrap)); // wait for thread to set promise OMNITRACE_DEBUG("waiting for child to signal it is setup...\n"); _fut.wait(); + stop_bundle(_bundle, threading::get_id()); + // unblock the signals in the entire process OMNITRACE_DEBUG("unblocking signals...\n"); tim::sampling::unblock_signals({ SIGALRM, SIGPROF }, diff --git a/projects/rocprofiler-systems/src/library/components/rocm_smi.cpp b/projects/rocprofiler-systems/src/library/components/rocm_smi.cpp new file mode 100644 index 0000000000..0effc359dc --- /dev/null +++ b/projects/rocprofiler-systems/src/library/components/rocm_smi.cpp @@ -0,0 +1,569 @@ +// Copyright (c) 2018 Advanced Micro Devices, Inc. All Rights Reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// with the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in the +// documentation and/or other materials provided with the distribution. +// +// * Neither the names of Advanced Micro Devices, Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this Software without specific prior written permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH +// THE SOFTWARE. + +#if defined(NDEBUG) +# undef NDEBUG +#endif + +#include "library/components/rocm_smi.hpp" +#include "library/common.hpp" +#include "library/components/fwd.hpp" +#include "library/config.hpp" +#include "library/critical_trace.hpp" +#include "library/debug.hpp" +#include "library/gpu.hpp" +#include "library/perfetto.hpp" + +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +#define OMNITRACE_ROCM_SMI_CALL(ERROR_CODE) ::omnitrace::rocm_smi::check_error(ERROR_CODE) + +namespace omnitrace +{ +namespace rocm_smi +{ +using tim::type_mutex; +using auto_lock_t = tim::auto_lock_t; + +namespace +{ +bool& +is_initialized() +{ + static bool _v = false; + return _v; +} + +void +check_error(rsmi_status_t ec) +{ + if(ec == RSMI_STATUS_SUCCESS) return; + const char* _msg = nullptr; + auto _err = rsmi_status_string(ec, &_msg); + if(_err != RSMI_STATUS_SUCCESS) + OMNITRACE_THROW( + "rsmi_status_string(%i, ...) failed. No error message available\n", (int) ec); + OMNITRACE_THROW("%s", _msg); +} + +std::atomic& +get_rocm_smi_state() +{ + static std::atomic _v{ State::PreInit }; + return _v; +} +} // namespace + +//--------------------------------------------------------------------------------------// + +size_t data::device_count = 0; +std::set data::device_list = {}; +std::unique_ptr data::polling_finished = {}; + +data::data(uint32_t _dev_id) { sample(_dev_id); } + +void +data::sample(uint32_t _dev_id) +{ + auto _ts = tim::get_clock_real_now(); + assert(_ts < std::numeric_limits::max()); + + m_dev_id = _dev_id; + m_ts = _ts; + + rsmi_dev_busy_percent_get(_dev_id, &m_busy_perc); + rsmi_dev_temp_metric_get(_dev_id, RSMI_TEMP_TYPE_EDGE, RSMI_TEMP_CURRENT, &m_temp); + rsmi_dev_power_ave_get(_dev_id, 0, &m_power); + rsmi_dev_memory_usage_get(_dev_id, RSMI_MEM_TYPE_VRAM, &m_mem_usage); +} + +void +data::print(std::ostream& _os) const +{ + std::stringstream _ss{}; + _ss << "device: " << m_dev_id << ", busy = " << m_busy_perc << "%, temp = " << m_temp + << ", power = " << m_power << ", memory usage = " << m_mem_usage; + _os << _ss.str(); +} + +namespace +{ +struct cpu_freq +{}; +using freq_pair_t = std::pair; +std::vector> cpu_frequencies = {}; + +struct cpu_mem +{}; +using cpu_mem_usage_pair_t = std::pair; +std::vector cpu_mem_usage = {}; +} // namespace + +void +data::poll(std::atomic* _state, nsec_t _interval, promise_t* _ready) +{ + threading::set_thread_name("omni.rocm_smi"); + + // notify thread started + if(_ready) _ready->set_value(); + + std::vector*> _bundle_data{}; + _bundle_data.resize(device_count, nullptr); + for(size_t i = 0; i < device_count; ++i) + { + if(device_list.count(i) > 0) + { + _bundle_data.at(i) = &sampler_instances::instances().at(i); + if(!*_bundle_data.at(i)) *_bundle_data.at(i) = std::make_unique(); + } + } + + auto _ncpu = threading::affinity::hw_concurrency(); + std::vector _cpu_mhz_pos{}; + std::ifstream _ifs{ "/proc/cpuinfo" }; + if(_ifs) + { + for(size_t i = 0; i < _ncpu; ++i) + { + short _n = 0; + std::string _st{}; + while(_ifs && _ifs.good()) + { + std::string _s{}; + _ifs >> _s; + if(!_ifs.good() || !_ifs) break; + + if(_s == "cpu" || _s == "MHz" || _s == ":") + { + ++_n; + _st += _s + " "; + } + else + { + _n = 0; + _st = {}; + } + + if(_n == 3) + { + size_t _pos = _ifs.tellg(); + _cpu_mhz_pos.emplace_back(_pos + 1); + _ifs >> _s; + if(!_ifs.good() || !_ifs) break; + OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug() || get_verbose() > 1, + "[%zu] %s %s (pos = %zu)\n", i, + _st.c_str(), _s.c_str(), _pos + 1); + break; + } + } + } + } + + cpu_frequencies.resize(_ncpu); + + _ifs = std::ifstream{ "/proc/cpuinfo", std::ifstream::binary }; + auto _read_cpu_freq = [&_cpu_mhz_pos, &_ifs](size_t _idx) { + double _freq = 0; + _ifs.seekg(_cpu_mhz_pos.at(_idx), _ifs.beg); + _ifs >> _freq; + return _freq; + }; + + auto _read_cpu_mem_usage = []() { + cpu_mem_usage.emplace_back(tim::get_clock_real_now(), + tim::get_page_rss()); + }; + + auto _read_cpu_freqs = [&_read_cpu_freq, _ncpu]() { + auto _ts = tim::get_clock_real_now(); + for(size_t i = 0; i < _ncpu; ++i) + { + auto _freq = _read_cpu_freq(i); + cpu_frequencies.at(i).emplace_back(_ts, _freq); + } + }; + + OMNITRACE_CONDITIONAL_BASIC_PRINT( + get_verbose() > 0 || get_debug(), + "Polling rocm-smi for %zu device(s) at an interval of %f seconds...\n", + device_list.size(), + std::chrono::duration_cast>(_interval).count()); + + get_initial().resize(device_count); + for(auto itr : device_list) + get_initial().at(itr).sample(itr); + + auto _now = std::chrono::steady_clock::now(); + while(_state && _state->load() != State::Finalized && get_state() != State::Finalized) + { + std::this_thread::sleep_until(_now); + if(_state->load() != State::Active) continue; + _read_cpu_mem_usage(); + _read_cpu_freqs(); + for(auto itr : device_list) + { + OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug(), + "Polling rocm-smi for device %u...\n", itr); + auto& _data = *_bundle_data.at(itr); + if(!_data) continue; + _data->emplace_back(data{ itr }); + OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug(), " %s\n", + TIMEMORY_JOIN("", _data->back()).c_str()); + } + while(_now < std::chrono::steady_clock::now()) + _now += _interval; + } + OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug(), "Polling rocm-smi completed...\n"); + + if(polling_finished) polling_finished->set_value(); +} + +std::vector& +data::get_initial() +{ + static std::vector _v{}; + return _v; +} + +std::unique_ptr& +data::get_thread() +{ + static std::unique_ptr _v; + return _v; +} + +void +data::set_state(State _state) +{ + get_rocm_smi_state().store(_state); +} + +bool +data::setup() +{ + perfetto_counter_track::init(); + perfetto_counter_track::init(); + perfetto_counter_track::init(); + + // shutdown if already running + shutdown(); + + OMNITRACE_DEBUG("Configuring rocm-smi...\n"); + + auto _freq = get_rocm_smi_freq(); + uint64_t _msec_freq = (1.0 / _freq) * 1.0e3; + + promise_t _prom{}; + auto _fut = _prom.get_future(); + polling_finished = std::make_unique(); + + set_state(State::PreInit); + get_thread() = std::make_unique( + &data::poll, &get_rocm_smi_state(), msec_t{ _msec_freq }, &_prom); + + _fut.wait(); + return true; +} + +bool +data::shutdown() +{ + auto& _thread = get_thread(); + if(_thread) + { + OMNITRACE_DEBUG("Shutting down rocm-smi...\n"); + set_state(State::Finalized); + if(polling_finished) + { + auto _fut = polling_finished->get_future(); + uint64_t _freq = (1.0 / get_rocm_smi_freq()) * 1.0e3; + _fut.wait_for(msec_t{ 5 * _freq }); + _thread->join(); + } + else + { + uint64_t _freq = (1.0 / get_rocm_smi_freq()) * 1.0e3; + std::this_thread::sleep_for(msec_t{ 5 * _freq }); + pthread_cancel(_thread->native_handle()); + _thread->detach(); + } + _thread = std::unique_ptr{}; + polling_finished = std::unique_ptr{}; + return true; + } + return false; +} + +#define GPU_METRIC(COMPONENT, ...) \ + if constexpr(tim::trait::is_available::value) \ + { \ + auto* _val = _v.get(); \ + if(_val) \ + { \ + _val->set_value(itr.__VA_ARGS__); \ + _val->set_accum(itr.__VA_ARGS__); \ + } \ + } + +void +data::post_process(uint32_t _dev_id) +{ + OMNITRACE_CONDITIONAL_PRINT(get_debug() || get_verbose() > 0, + "Post-processing rocm-smi data for device %u\n", _dev_id); + + using component::sampling_gpu_busy; + using component::sampling_gpu_memory; + using component::sampling_gpu_power; + using component::sampling_gpu_temp; + using bundle_t = tim::lightweight_tuple; + + auto _process_frequencies = [](size_t _idx) { + using counter_track = perfetto_counter_track; + if(!counter_track::exists(_idx)) + { + auto _devname = TIMEMORY_JOIN("", "[CPU ", _idx, "] "); + auto addendum = [&](const char* _v) { return _devname + std::string{ _v }; }; + counter_track::emplace(_idx, addendum("Frequency (S)"), "MHz"); + } + + for(auto& itr : cpu_frequencies.at(_idx)) + { + uint64_t _ts = itr.first; + double _freq = itr.second; + TRACE_COUNTER("sampling", counter_track::at(_idx, 0), _ts, _freq); + } + }; + + auto _process_cpu_mem_usage = []() { + using counter_track = perfetto_counter_track; + if(!counter_track::exists(0)) + { + auto _devname = TIMEMORY_JOIN("", "[CPU] "); + auto addendum = [&](const char* _v) { return _devname + std::string{ _v }; }; + counter_track::emplace(0, addendum("Memory Usage (S)"), "MB"); + } + + for(auto& itr : cpu_mem_usage) + { + uint64_t _ts = itr.first; + double _mem_usage = itr.second; + TRACE_COUNTER("sampling", counter_track::at(0, 0), _ts, + _mem_usage / units::megabyte); + } + }; + + static bool _once = false; + if(!_once) + { + _once = true; + _process_cpu_mem_usage(); + for(size_t i = 0; i < cpu_frequencies.size(); ++i) + _process_frequencies(i); + } + + if(device_count < _dev_id) return; + + auto& _rocm_smi_v = sampler_instances::instances().at(_dev_id); + auto _rocm_smi = (_rocm_smi_v) ? *_rocm_smi_v : std::deque{}; + + auto _process_perfetto = [&]() { + for(auto& itr : _rocm_smi) + { + using counter_track = perfetto_counter_track; + if(itr.m_dev_id != _dev_id) continue; + if(!counter_track::exists(_dev_id)) + { + auto _devname = TIMEMORY_JOIN("", "[GPU ", _dev_id, "] "); + auto addendum = [&](const char* _v) { + return _devname + std::string{ _v }; + }; + counter_track::emplace(_dev_id, addendum("Busy"), "%"); + counter_track::emplace(_dev_id, addendum("Temperature"), "deg C"); + counter_track::emplace(_dev_id, addendum("Power"), "watts"); + counter_track::emplace(_dev_id, addendum("Memory Usage"), "megabytes"); + } + uint64_t _ts = itr.m_ts; + double _busy = itr.m_busy_perc; + double _temp = itr.m_temp / 1.0e3; + double _power = itr.m_power / 1.0e6; + double _usage = itr.m_mem_usage / static_cast(units::megabyte); + TRACE_COUNTER("rocm_smi", counter_track::at(_dev_id, 0), _ts, _busy); + TRACE_COUNTER("rocm_smi", counter_track::at(_dev_id, 1), _ts, _temp); + TRACE_COUNTER("rocm_smi", counter_track::at(_dev_id, 2), _ts, _power); + TRACE_COUNTER("rocm_smi", counter_track::at(_dev_id, 3), _ts, _usage); + } + }; + + if(get_use_perfetto()) _process_perfetto(); + + if(!get_use_timemory()) return; + + for(auto& itr : _rocm_smi) + { + using entry_t = critical_trace::entry; + auto _ts = itr.m_ts; + auto _entries = critical_trace::get_entries(_ts, [](const entry_t& _e) { + return _e.device == critical_trace::Device::GPU; + }); + + std::vector _tc{}; + _tc.reserve(_entries.size()); + for(auto& eitr : _entries) + { + auto& _v = _tc.emplace_back(eitr.first); + _v.push(); + _v.start(); + _v.stop(); + + GPU_METRIC(sampling_gpu_busy, m_busy_perc) + GPU_METRIC(sampling_gpu_temp, m_temp / 1.0e3) // provided in milli-degree C + GPU_METRIC(sampling_gpu_power, + m_power * units::microwatt / static_cast(units::watt)) + GPU_METRIC(sampling_gpu_memory, + m_mem_usage / static_cast(units::megabyte)) + + _v.pop(); + } + } +} + +//--------------------------------------------------------------------------------------// + +void +setup() +{ + auto_lock_t _lk{ type_mutex() }; + + if(is_initialized() || !get_use_rocm_smi()) return; + + pthread_gotcha::enable_sampling_on_child_threads() = false; + + // assign the data value to determined by rocm-smi + data::device_count = device_count(); + + auto _devices_v = get_rocm_smi_devices(); + for(auto& itr : _devices_v) + itr = tolower(itr); + bool _all_devices = _devices_v.find("all") != std::string::npos || _devices_v.empty(); + bool _no_devices = _devices_v.find("none") != std::string::npos; + + std::set _devices{}; + if(_all_devices) + { + for(uint32_t i = 0; i < data::device_count; ++i) + _devices.emplace(i); + } + else if(!_no_devices) + { + for(auto&& itr : tim::delimit(get_rocm_smi_devices())) + { + uint32_t idx = std::stoul(itr); + if(idx < data::device_count) _devices.emplace(idx); + } + } + + data::device_list = _devices; + + for(auto itr : _devices) + { + uint16_t dev_id = 0; + OMNITRACE_ROCM_SMI_CALL(rsmi_dev_id_get(itr, &dev_id)); + // dev_id holds the device ID of device i, upon a successful call + } + + is_initialized() = true; + + data::setup(); + + pthread_gotcha::enable_sampling_on_child_threads() = true; + omnitrace::rocm_smi::data::set_state(State::Active); +} + +void +shutdown() +{ + auto_lock_t _lk{ type_mutex() }; + + if(!is_initialized()) return; + + if(data::shutdown()) + { + OMNITRACE_ROCM_SMI_CALL(rsmi_shut_down()); + } + + is_initialized() = false; +} + +uint32_t +device_count() +{ + uint32_t _num_devices = 0; + try + { + static auto _rsmi_init_once = []() { OMNITRACE_ROCM_SMI_CALL(rsmi_init(0)); }; + static std::once_flag _once{}; + std::call_once(_once, _rsmi_init_once); + + OMNITRACE_ROCM_SMI_CALL(rsmi_num_monitor_devices(&_num_devices)); + } catch(const std::exception& _e) + { + OMNITRACE_BASIC_PRINT("Exception: %s\n", _e.what()); + } + return _num_devices; +} +} // namespace rocm_smi +} // namespace omnitrace + +TIMEMORY_INSTANTIATE_EXTERN_COMPONENT( + TIMEMORY_ESC(data_tracker), true, + double) + +TIMEMORY_INSTANTIATE_EXTERN_COMPONENT( + TIMEMORY_ESC(data_tracker), true, + double) + +TIMEMORY_INSTANTIATE_EXTERN_COMPONENT( + TIMEMORY_ESC(data_tracker), true, + double) + +TIMEMORY_INSTANTIATE_EXTERN_COMPONENT( + TIMEMORY_ESC(data_tracker), true, + double) diff --git a/projects/rocprofiler-systems/src/library/components/roctracer.cpp b/projects/rocprofiler-systems/src/library/components/roctracer.cpp index 2a1f862c86..0d36d111bf 100644 --- a/projects/rocprofiler-systems/src/library/components/roctracer.cpp +++ b/projects/rocprofiler-systems/src/library/components/roctracer.cpp @@ -21,11 +21,16 @@ // SOFTWARE. #include "library/components/roctracer.hpp" +#include "library/components/pthread_gotcha.hpp" +#include "library/components/rocm_smi.hpp" #include "library/components/roctracer_callbacks.hpp" #include "library/config.hpp" #include "library/defines.hpp" +#include "library/redirect.hpp" +#include "library/sampling.hpp" #include "library/thread_data.hpp" +namespace rocm_smi = omnitrace::rocm_smi; using namespace omnitrace; namespace tim @@ -35,11 +40,22 @@ namespace component void roctracer::preinit() { - OMNITRACE_DEBUG("[%s]\n", __FUNCTION__); roctracer_data::label() = "roctracer"; roctracer_data::description() = "ROCm tracer (activity API)"; } +void +roctracer::start() +{ + if(tracker_type::start() == 0) setup(); +} + +void +roctracer::stop() +{ + if(tracker_type::stop() == 0) shutdown(); +} + bool roctracer::is_setup() { @@ -53,9 +69,9 @@ roctracer::add_setup(const std::string& _lbl, std::function&& _func) } void -roctracer::add_tear_down(const std::string& _lbl, std::function&& _func) +roctracer::add_shutdown(const std::string& _lbl, std::function&& _func) { - roctracer_tear_down_routines().emplace_back(_lbl, std::move(_func)); + roctracer_shutdown_routines().emplace_back(_lbl, std::move(_func)); } void @@ -73,7 +89,7 @@ roctracer::remove_setup(const std::string& _lbl) } void -roctracer::remove_tear_down(const std::string& _lbl) +roctracer::remove_shutdown(const std::string& _lbl) { auto& _data = roctracer_setup_routines(); for(auto itr = _data.begin(); itr != _data.end(); ++itr) @@ -89,31 +105,44 @@ roctracer::remove_tear_down(const std::string& _lbl) void roctracer::setup() { - if(!get_use_timemory() && !get_use_perfetto()) return; + trait::runtime_enabled::set(get_use_timemory()); + + if(!get_use_roctracer()) return; auto_lock_t _lk{ type_mutex() }; if(roctracer_is_setup()) return; roctracer_is_setup() = true; + OMNITRACE_DEBUG("[%s]\n", __FUNCTION__); - tim::set_env("HSA_TOOLS_LIB", "libomnitrace.so", 0); - +#if OMNITRACE_HIP_VERSION_MAJOR == 4 && OMNITRACE_HIP_VERSION_MINOR < 4 auto _kfdwrapper = dynamic_library{ "OMNITRACE_ROCTRACER_LIBKFDWRAPPER", OMNITRACE_ROCTRACER_LIBKFDWRAPPER }; +#endif - ROCTRACER_CALL(roctracer_set_properties(ACTIVITY_DOMAIN_HIP_API, nullptr)); + ROCTRACER_CALL( + roctracer_set_properties(ACTIVITY_DOMAIN_HIP_API, (void*) hip_api_callback)); - // if(roctracer_default_pool() == nullptr) + if(roctracer_default_pool() == nullptr) { // Allocating tracing pool roctracer_properties_t properties{}; memset(&properties, 0, sizeof(roctracer_properties_t)); - properties.mode = 0x1000; + // properties.mode = 0x1000; properties.buffer_size = 0x1000; properties.buffer_callback_fun = hip_activity_callback; ROCTRACER_CALL(roctracer_open_pool(&properties)); } +#if OMNITRACE_HIP_VERSION_MAJOR == 4 && OMNITRACE_HIP_VERSION_MINOR >= 4 && \ + OMNITRACE_HIP_VERSION_MINOR <= 5 + // HIP 4.5.0 has an invalid warning + redirect _rd{ std::cerr, "roctracer_enable_callback(), get_op_end(), invalid domain " + "ID(4) in: roctracer_enable_callback(hip_api_callback, " + "nullptr)roctracer_enable_activity_expl(), get_op_end(), " + "invalid domain ID(4) in: roctracer_enable_activity()" }; +#endif + // Enable API callbacks, all domains ROCTRACER_CALL(roctracer_enable_callback(hip_api_callback, nullptr)); // Enable activity tracing, all domains @@ -125,7 +154,7 @@ roctracer::setup() } void -roctracer::tear_down() +roctracer::shutdown() { auto_lock_t _lk{ type_mutex() }; if(!roctracer_is_setup()) return; @@ -133,37 +162,48 @@ roctracer::tear_down() OMNITRACE_DEBUG("[%s]\n", __FUNCTION__); // flush all the activity - // if(roctracer_default_pool() != nullptr) + if(roctracer_default_pool() != nullptr) { + OMNITRACE_DEBUG("[%s] roctracer_flush_activity\n", __FUNCTION__); ROCTRACER_CALL(roctracer_flush_activity()); - } - // flush all buffers - roctracer_flush_buf(); + // flush all buffers + OMNITRACE_DEBUG("[%s] roctracer_flush_buf\n", __FUNCTION__); + roctracer_flush_buf(); + } + + OMNITRACE_DEBUG("[%s] executing hip_exec_activity_callbacks\n", __FUNCTION__); // make sure all async operations are executed for(size_t i = 0; i < max_supported_threads; ++i) hip_exec_activity_callbacks(i); // callback for hsa - for(auto& itr : roctracer_tear_down_routines()) + OMNITRACE_DEBUG("[%s] executing roctracer_shutdown_routines...\n", __FUNCTION__); + for(auto& itr : roctracer_shutdown_routines()) itr.second(); +#if OMNITRACE_HIP_VERSION_MAJOR == 4 && OMNITRACE_HIP_VERSION_MINOR >= 4 && \ + OMNITRACE_HIP_VERSION_MINOR <= 5 + OMNITRACE_DEBUG("[%s] redirecting roctracer warnings\n", __FUNCTION__); + // HIP 4.5.0 has an invalid warning + redirect _rd{ + std::cerr, "roctracer_disable_callback(), get_op_end(), invalid domain ID(4) " + "in: roctracer_disable_callback()roctracer_disable_activity(), " + "get_op_end(), invalid domain ID(4) in: roctracer_disable_activity()" + }; +#endif + // Disable tracing and closing the pool + OMNITRACE_DEBUG("[%s] roctracer_disable_callback\n", __FUNCTION__); ROCTRACER_CALL(roctracer_disable_callback()); + + OMNITRACE_DEBUG("[%s] roctracer_disable_activity\n", __FUNCTION__); ROCTRACER_CALL(roctracer_disable_activity()); + + OMNITRACE_DEBUG("[%s] roctracer_close_pool\n", __FUNCTION__); ROCTRACER_CALL(roctracer_close_pool()); -} -void -roctracer::start() -{ - if(tracker_type::start() == 0) setup(); -} - -void -roctracer::stop() -{ - if(tracker_type::stop() == 0) tear_down(); + OMNITRACE_DEBUG("[%s] roctracer is shutdown\n", __FUNCTION__); } } // namespace component } // namespace tim @@ -181,92 +221,103 @@ extern "C" bool OnLoad(HsaApiTable* table, uint64_t runtime_version, uint64_t failed_tool_count, const char* const* failed_tool_names) { - OMNITRACE_DEBUG("[%s]\n", __FUNCTION__); + pthread_gotcha::enable_sampling_on_child_threads() = false; + OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug_env() || get_verbose_env() > 0, + "[%s]\n", __FUNCTION__); tim::consume_parameters(table, runtime_version, failed_tool_count, failed_tool_names); - // ONLOAD_TRACE_BEG(); - // on_exit(exit_handler, nullptr); - auto _setup = [=]() { - get_hsa_timer() = - std::make_unique(table->core_->hsa_system_get_info_fn); - - // const char* output_prefix = getenv("ROCP_OUTPUT_DIR"); - const char* output_prefix = nullptr; - - // App begin timestamp begin_ts_file.txt - // begin_ts_file_handle = open_output_file(output_prefix, - // "begin_ts_file.txt"); const timestamp_t app_start_time = - // timer->timestamp_fn_ns(); fprintf(begin_ts_file_handle, "%lu\n", - // app_start_time); - - bool trace_hsa_api = get_trace_hsa_api(); - std::vector hsa_api_vec = - tim::delimit(get_trace_hsa_api_types()); - - // Enable HSA API callbacks/activity - if(trace_hsa_api) + try { - // hsa_api_file_handle = open_output_file(output_prefix, - // "hsa_api_trace.txt"); + OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug() || get_verbose() > 1, + "[%s] setting up HSA...\n", + __FUNCTION__); - // initialize HSA tracing - roctracer_set_properties(ACTIVITY_DOMAIN_HSA_API, (void*) table); + get_hsa_timer() = + std::make_unique(table->core_->hsa_system_get_info_fn); - OMNITRACE_DEBUG(" HSA-trace("); - if(!hsa_api_vec.empty()) + // const char* output_prefix = getenv("ROCP_OUTPUT_DIR"); + const char* output_prefix = nullptr; + + bool trace_hsa_api = get_trace_hsa_api(); + + // Enable HSA API callbacks/activity + if(trace_hsa_api) { - for(const auto& itr : hsa_api_vec) + std::vector hsa_api_vec = + tim::delimit(get_trace_hsa_api_types()); + + // initialize HSA tracing + roctracer_set_properties(ACTIVITY_DOMAIN_HSA_API, (void*) table); + + OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug() || get_verbose() > 1, + " HSA-trace("); + if(!hsa_api_vec.empty()) { - uint32_t cid = HSA_API_ID_NUMBER; - const char* api = itr.c_str(); - ROCTRACER_CALL(roctracer_op_code(ACTIVITY_DOMAIN_HSA_API, api, - &cid, nullptr)); - ROCTRACER_CALL(roctracer_enable_op_callback( - ACTIVITY_DOMAIN_HSA_API, cid, hsa_api_callback, nullptr)); + for(const auto& itr : hsa_api_vec) + { + uint32_t cid = HSA_API_ID_NUMBER; + const char* api = itr.c_str(); + ROCTRACER_CALL(roctracer_op_code(ACTIVITY_DOMAIN_HSA_API, api, + &cid, nullptr)); + ROCTRACER_CALL(roctracer_enable_op_callback( + ACTIVITY_DOMAIN_HSA_API, cid, hsa_api_callback, nullptr)); - OMNITRACE_DEBUG(" %s", api); + OMNITRACE_CONDITIONAL_BASIC_PRINT( + get_debug() || get_verbose() > 1, " %s", api); + } } + else + { + ROCTRACER_CALL(roctracer_enable_domain_callback( + ACTIVITY_DOMAIN_HSA_API, hsa_api_callback, nullptr)); + } + OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug() || get_verbose() > 1, + "\n"); } - else + + bool trace_hsa_activity = get_trace_hsa_activity(); + // Enable HSA GPU activity + if(trace_hsa_activity) { - ROCTRACER_CALL(roctracer_enable_domain_callback( - ACTIVITY_DOMAIN_HSA_API, hsa_api_callback, nullptr)); + // initialize HSA tracing + ::roctracer::hsa_ops_properties_t ops_properties{ + table, + reinterpret_cast( + hsa_activity_callback), + nullptr, output_prefix + }; + roctracer_set_properties(ACTIVITY_DOMAIN_HSA_OPS, &ops_properties); + + OMNITRACE_CONDITIONAL_BASIC_PRINT(get_debug() || get_verbose() > 1, + " HSA-activity-trace()\n"); + ROCTRACER_CALL(roctracer_enable_op_activity(ACTIVITY_DOMAIN_HSA_OPS, + HSA_OP_ID_COPY)); } - OMNITRACE_DEBUG("\n"); - } - - bool trace_hsa_activity = get_trace_hsa_activity(); - // Enable HSA GPU activity - if(trace_hsa_activity) + } catch(std::exception& _e) { - // initialize HSA tracing - ::roctracer::hsa_ops_properties_t ops_properties{ - table, - reinterpret_cast(hsa_activity_callback), - nullptr, output_prefix - }; - roctracer_set_properties(ACTIVITY_DOMAIN_HSA_OPS, &ops_properties); - - OMNITRACE_DEBUG(" HSA-activity-trace()\n"); - ROCTRACER_CALL(roctracer_enable_op_activity(ACTIVITY_DOMAIN_HSA_OPS, - HSA_OP_ID_COPY)); + OMNITRACE_BASIC_PRINT("Exception was thrown in HSA setup: %s\n", + _e.what()); } }; - auto _tear_down = []() { + auto _shutdown = []() { + OMNITRACE_DEBUG("[%s] roctracer_disable_domain_callback\n", __FUNCTION__); ROCTRACER_CALL(roctracer_disable_domain_callback(ACTIVITY_DOMAIN_HSA_API)); + OMNITRACE_DEBUG("[%s] roctracer_disable_op_activity\n", __FUNCTION__); ROCTRACER_CALL( roctracer_disable_op_activity(ACTIVITY_DOMAIN_HSA_OPS, HSA_OP_ID_COPY)); }; - if(comp::roctracer::is_setup()) _setup(); - comp::roctracer::add_setup("hsa", std::move(_setup)); - comp::roctracer::add_tear_down("hsa", std::move(_tear_down)); + comp::roctracer::add_shutdown("hsa", std::move(_shutdown)); + rocm_smi::setup(); + comp::roctracer::setup(); + + pthread_gotcha::enable_sampling_on_child_threads() = true; return true; } @@ -274,6 +325,7 @@ extern "C" void OnUnload() { OMNITRACE_DEBUG("[%s]\n", __FUNCTION__); - // ONLOAD_TRACE(""); + rocm_smi::shutdown(); + comp::roctracer::shutdown(); } } diff --git a/projects/rocprofiler-systems/src/library/components/roctracer_callbacks.cpp b/projects/rocprofiler-systems/src/library/components/roctracer_callbacks.cpp index fe66aaa5c3..0b04be6f22 100644 --- a/projects/rocprofiler-systems/src/library/components/roctracer_callbacks.cpp +++ b/projects/rocprofiler-systems/src/library/components/roctracer_callbacks.cpp @@ -24,6 +24,7 @@ #include "library.hpp" #include "library/config.hpp" #include "library/critical_trace.hpp" +#include "library/sampling.hpp" #include "library/thread_data.hpp" #include @@ -46,7 +47,7 @@ auto& get_roctracer_hip_data(int64_t _tid = threading::get_id()) { using data_t = std::unordered_map; - using thread_data_t = omnitrace_thread_data; + using thread_data_t = thread_data; static auto& _v = thread_data_t::instances(thread_data_t::construct_on_init{}); return _v.at(_tid); } @@ -76,9 +77,8 @@ get_roctracer_cid_data() auto& get_hip_activity_callbacks(int64_t _tid = threading::get_id()) { - using thread_data_t = - omnitrace_thread_data>, api::roctracer>; - static auto& _v = thread_data_t::instances(thread_data_t::construct_on_init{}); + using thread_data_t = thread_data>, api::roctracer>; + static auto& _v = thread_data_t::instances(thread_data_t::construct_on_init{}); return _v.at(_tid); } @@ -214,11 +214,17 @@ hsa_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void* void hsa_activity_callback(uint32_t op, activity_record_t* record, void* arg) { + if(get_state() != State::Active || !trait::runtime_enabled::get()) + return; + static const char* copy_op_name = "hsa_async_copy"; static const char* dispatch_op_name = "hsa_dispatch"; static const char* barrier_op_name = "hsa_barrier"; const char** _name = nullptr; + static thread_local auto _once = (threading::set_thread_name("omni.roctracer"), true); + (void) _once; + switch(op) { case HSA_OP_ID_DISPATCH: _name = &dispatch_op_name; break; @@ -318,7 +324,7 @@ hip_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void* data->args.hipLaunchKernel.stream); if(_name != nullptr) { - if(get_use_perfetto() || get_use_timemory()) + if(get_use_perfetto() || get_use_timemory() || get_use_rocm_smi()) { tim::auto_lock_t _lk{ tim::type_mutex() }; get_roctracer_key_data().emplace(data->correlation_id, _name); @@ -333,7 +339,7 @@ hip_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void* const char* _name = hipKernelNameRef(data->args.hipModuleLaunchKernel.f); if(_name != nullptr) { - if(get_use_perfetto() || get_use_timemory()) + if(get_use_perfetto() || get_use_timemory() || get_use_rocm_smi()) { tim::auto_lock_t _lk{ tim::type_mutex() }; get_roctracer_key_data().emplace(data->correlation_id, _name); @@ -345,7 +351,7 @@ hip_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void* } default: { - if(get_use_perfetto() || get_use_timemory()) + if(get_use_perfetto() || get_use_timemory() || get_use_rocm_smi()) { tim::auto_lock_t _lk{ tim::type_mutex() }; get_roctracer_key_data().emplace(data->correlation_id, op_name); @@ -373,7 +379,7 @@ hip_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void* get_roctracer_hip_data()->erase(itr.first); } } - if(get_use_critical_trace()) + if(get_use_critical_trace() || get_use_rocm_smi()) { auto _cid = get_cpu_cid()++; uint16_t _depth = (get_cpu_cid_stack()->empty()) @@ -422,7 +428,7 @@ hip_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void* } } } - if(get_use_critical_trace()) + if(get_use_critical_trace() || get_use_rocm_smi()) { uint16_t _depth = 0; uint64_t _cid = 0; @@ -445,6 +451,14 @@ hip_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void* void hip_activity_callback(const char* begin, const char* end, void*) { + if(get_state() != State::Active || !trait::runtime_enabled::get()) + return; + + sampling::block_signals(); + + static thread_local auto _once = (threading::set_thread_name("omni.roctracer"), true); + (void) _once; + using Device = critical_trace::Device; using Phase = critical_trace::Phase; @@ -494,7 +508,7 @@ hip_activity_callback(const char* begin, const char* end, void*) auto _laps = _indexes[_corr_id]++; // see note #1 const char* _name = nullptr; bool _found = false; - bool _critical_trace = get_use_critical_trace(); + bool _critical_trace = get_use_critical_trace() || get_use_rocm_smi(); { tim::auto_lock_t _lk{ tim::type_mutex() }; @@ -597,7 +611,7 @@ roctracer_setup_routines() } roctracer_functions_t& -roctracer_tear_down_routines() +roctracer_shutdown_routines() { static auto _v = roctracer_functions_t{}; return _v; diff --git a/projects/rocprofiler-systems/src/library/config.cpp b/projects/rocprofiler-systems/src/library/config.cpp index 226e7bf2c0..cff5ab2c99 100644 --- a/projects/rocprofiler-systems/src/library/config.cpp +++ b/projects/rocprofiler-systems/src/library/config.cpp @@ -54,12 +54,30 @@ get_config() (void) _once; } +std::string +get_setting_name(std::string _v) +{ + static const auto _prefix = tim::string_view_t{ "omnitrace_" }; + for(auto& itr : _v) + itr = tolower(itr); + auto _pos = _v.find(_prefix); + if(_pos == 0) return _v.substr(_prefix.length()); + return _v; +} + #define OMNITRACE_CONFIG_SETTING(TYPE, ENV_NAME, DESCRIPTION, INITIAL_VALUE, ...) \ - _config->insert( \ - ENV_NAME, ENV_NAME, DESCRIPTION, INITIAL_VALUE, \ - std::set{ "custom", "omnitrace", __VA_ARGS__ }) + { \ + auto _ret = _config->insert( \ + ENV_NAME, get_setting_name(ENV_NAME), DESCRIPTION, INITIAL_VALUE, \ + std::set{ "custom", "omnitrace", __VA_ARGS__ }); \ + if(!_ret.second) \ + OMNITRACE_PRINT("Warning! Duplicate setting: %s / %s\n", \ + get_setting_name(ENV_NAME).c_str(), ENV_NAME); \ + } } // namespace +inline namespace config +{ void configure_settings() { @@ -67,25 +85,24 @@ configure_settings() if(_once) return; _once = true; + OMNITRACE_CONDITIONAL_THROW( + get_state() < State::Init, + "config::configure_settings() called before omnitrace_init_library. state = %s", + std::to_string(get_state()).c_str()); + + OMNITRACE_CONDITIONAL_THROW( + get_state() > State::Init, + "config::configure_settings() called after omnitrace was initialized. state = %s", + std::to_string(get_state()).c_str()); + static auto _config = settings::shared_instance(); - // auto* _config = settings::instance(); // if using timemory, default to perfetto being off auto _default_perfetto_v = !tim::get_env("OMNITRACE_USE_TIMEMORY", false, false); - auto _default_config_file = - JOIN("/", tim::get_env("HOME", "."), "omnitrace.cfg"); - auto _system_backend = tim::get_env("OMNITRACE_BACKEND_SYSTEM", false, false); - OMNITRACE_CONFIG_SETTING(std::string, "OMNITRACE_CONFIG_FILE", - "Configuration file of omnitrace and timemory settings", - _default_config_file, "config"); - - OMNITRACE_CONFIG_SETTING(bool, "OMNITRACE_DEBUG", "Enable debugging output", - _config->get_debug(), "debugging"); - auto _omnitrace_debug = _config->get("OMNITRACE_DEBUG"); if(_omnitrace_debug) tim::set_env("TIMEMORY_DEBUG_SETTINGS", "1", 0); @@ -101,6 +118,22 @@ configure_settings() "backend", "roctracer"); #endif +#if defined(OMNITRACE_USE_ROCM_SMI) + OMNITRACE_CONFIG_SETTING( + bool, "OMNITRACE_USE_ROCM_SMI", + "Enable sampling GPU power, temp, utilization, and memory usage", true, "backend", + "rocm-smi"); + + OMNITRACE_CONFIG_SETTING(std::string, "OMNITRACE_ROCM_SMI_DEVICES", + "Devices to query when OMNITRACE_USE_ROCM_SMI=ON", "all", + "backend", "rocm-smi"); + + OMNITRACE_CONFIG_SETTING( + double, "OMNITRACE_ROCM_SMI_FREQ", + "Number of rocm-smi samples per second when OMNITTRACE_USE_ROCM_SMI=ON", 5.0, + "backend", "rocm-smi"); +#endif + OMNITRACE_CONFIG_SETTING(bool, "OMNITRACE_USE_SAMPLING", "Enable statistical sampling of call-stack", false, "backend", "sampling"); @@ -110,6 +143,9 @@ configure_settings() "Enable tagging filenames with process identifier (either MPI rank or pid)", true, "io"); + OMNITRACE_CONFIG_SETTING(bool, "OMNITRACE_USE_KOKKOSP", + "Enable support for Kokkos Tools", true, "kokkos"); + OMNITRACE_CONFIG_SETTING(size_t, "OMNITRACE_INSTRUMENTATION_INTERVAL", "Instrumentation only takes measurements once every N " "function calls (not statistical)", @@ -225,6 +261,40 @@ configure_settings() _config->get_stack_clearing() = false; _config->get_time_output() = true; _config->get_timing_precision() = 6; + _config->get_max_thread_bookmarks() = 1; + _config->get_timing_units() = "sec"; + _config->get_memory_units() = "MB"; + _config->get_papi_events() = "PAPI_TOT_CYC, PAPI_TOT_INS"; + +#if defined(TIMEMORY_USE_PAPI) + int _paranoid = 2; + { + std::ifstream _fparanoid{ "/proc/sys/kernel/perf_event_paranoid" }; + if(_fparanoid) _fparanoid >> _paranoid; + } + + if(_paranoid > 1) + { + OMNITRACE_CONDITIONAL_BASIC_PRINT( + get_verbose_env() >= 0, + "/proc/sys/kernel/perf_event_paranoid has a value of %i. " + "Disabling PAPI (requires a value <= 1)...\n", + _paranoid); + OMNITRACE_CONDITIONAL_BASIC_PRINT( + get_verbose_env() >= 0, + "In order to enable PAPI support, run 'echo N | sudo tee " + "/proc/sys/kernel/perf_event_paranoid' where N is < 2\n"); + tim::trait::runtime_enabled::set(false); + tim::trait::runtime_enabled::set(false); + tim::trait::runtime_enabled::set(false); + tim::trait::runtime_enabled::set(false); + tim::trait::runtime_enabled::set(false); + tim::trait::runtime_enabled::set(false); + _config->get_papi_events() = ""; + } +#else + _config->get_papi_quiet() = true; +#endif for(auto&& itr : tim::delimit(_config->get("OMNITRACE_CONFIG_FILE"), ";:")) @@ -265,15 +335,39 @@ configure_settings() if(tim::mpi::is_initialized()) settings::default_process_suffix() = tim::mpi::rank(); #endif OMNITRACE_CONDITIONAL_BASIC_PRINT(get_verbose_env() > 0, "configuration complete\n"); + +#if defined(OMNITRACE_USE_ROCM_SMI) + auto _rsmi_freq = _config->find("OMNITRACE_ROCM_SMI_FREQ"); + if(_rsmi_freq != _config->end()) + { + double& _rsmi_freq_v = + static_cast&>(*_rsmi_freq->second).get(); + if(_rsmi_freq_v > 1000) _rsmi_freq_v = 1000.; + } +#endif } void -print_config_settings( +print_banner(std::ostream& _os) +{ + static const char* _banner = R"banner( + + ______ .___ ___. .__ __. __ .___________..______ ___ ______ _______ + / __ \ | \/ | | \ | | | | | || _ \ / \ / || ____| + | | | | | \ / | | \| | | | `---| |----`| |_) | / ^ \ | ,----'| |__ + | | | | | |\/| | | . ` | | | | | | / / /_\ \ | | | __| + | `--' | | | | | | |\ | | | | | | |\ \----./ _____ \ | `----.| |____ + \______/ |__| |__| |__| \__| |__| |__| | _| `._____/__/ \__\ \______||_______| + + )banner"; + _os << _banner << std::endl; +} + +void +print_settings( std::ostream& _ros, std::function&)>&& _filter) { - if(get_verbose() < 1) return; - OMNITRACE_CONDITIONAL_BASIC_PRINT(true, "configuration:\n"); std::stringstream _os{}; @@ -370,6 +464,61 @@ print_config_settings( _ros << _os.str() << std::flush; } +void +print_settings() +{ + if(dmp::rank() > 0) return; + + static std::set _sample_options = { + "OMNITRACE_SAMPLING_FREQ", "OMNITRACE_SAMPLING_DELAY", + "OMNITRACE_FLAT_SAMPLING", "OMNITRACE_TIMELINE_SAMPLING", + "OMNITRACE_FLAT_SAMPLING", "OMNITRACE_TIMELINE_SAMPLING", + }; + static std::set _perfetto_options = { + "OMNITRACE_OUTPUT_FILE", + "OMNITRACE_BACKEND", + "OMNITRACE_SHMEM_SIZE_HINT_KB", + "OMNITRACE_BUFFER_SIZE_KB", + }; + static std::set _timemory_options = { + "OMNITRACE_ROCTRACER_FLAT_PROFILE", "OMNITRACE_ROCTRACER_TIMELINE_PROFILE" + }; + // generic filter for filtering relevant options + auto _is_omnitrace_option = [](const auto& _v, const auto& _c) { + if(!get_use_roctracer() && _v.find("OMNITRACE_ROCTRACER_") == 0) return false; + if(!get_use_critical_trace() && _v.find("OMNITRACE_CRITICAL_TRACE_") == 0) + return false; + if(!get_use_perfetto() && _perfetto_options.count(_v) > 0) return false; + if(!get_use_timemory() && _timemory_options.count(_v) > 0) return false; + if(!get_use_sampling() && _sample_options.count(_v) > 0) return false; + const auto npos = std::string::npos; + if(_v.find("WIDTH") != npos || _v.find("SEPARATOR_FREQ") != npos || + _v.find("AUTO_OUTPUT") != npos || _v.find("DART_OUTPUT") != npos || + _v.find("FILE_OUTPUT") != npos || _v.find("PLOT_OUTPUT") != npos || + _v.find("FLAMEGRAPH_OUTPUT") != npos) + return false; + if(!_c.empty()) + { + if(_c.find("omnitrace") != _c.end()) return true; + if(_c.find("debugging") != _c.end() && _v.find("DEBUG") != npos) return true; + if(_c.find("config") != _c.end()) return true; + if(_c.find("dart") != _c.end()) return false; + if(_c.find("io") != _c.end() && _v.find("_OUTPUT") != npos) return true; + if(_c.find("format") != _c.end()) return true; + return false; + } + return (_v.find("OMNITRACE_") == 0); + }; + + tim::print_env(std::cerr, [_is_omnitrace_option](const std::string& _v) { + return _is_omnitrace_option(_v, std::set{}); + }); + + print_settings(std::cerr, _is_omnitrace_option); + + fprintf(stderr, "\n"); +} + std::string& get_exe_name() { @@ -384,12 +533,56 @@ get_config_file() return static_cast&>(*_v->second).get(); } +Mode +get_mode() +{ + static auto _v = []() { + auto _mode = tim::get_env_choice("OMNITRACE_MODE", "trace", + { "trace", "sampling" }); + if(_mode == "sampling") return Mode::Sampling; + return Mode::Trace; + }(); + return _v; +} + +bool& +is_attached() +{ + static bool _v = false; + return _v; +} + +bool& +is_binary_rewrite() +{ + static bool _v = false; + return _v; +} + bool get_debug_env() { return tim::get_env("OMNITRACE_DEBUG", false); } +bool +get_is_continuous_integration() +{ + return tim::get_env("OMNITRACE_CI", false); +} + +bool +get_debug_init() +{ + return tim::get_env("OMNITRACE_DEBUG_INIT", false); +} + +bool +get_debug_finalize() +{ + return tim::get_env("OMNITRACE_DEBUG_FINALIZE", false); +} + bool get_debug() { @@ -430,6 +623,18 @@ get_use_roctracer() #if defined(OMNITRACE_USE_ROCTRACER) static auto _v = get_config()->find("OMNITRACE_USE_ROCTRACER"); return static_cast&>(*_v->second).get(); +#else + static auto _v = false; + return _v; +#endif +} + +bool& +get_use_rocm_smi() +{ +#if defined(OMNITRACE_USE_ROCM_SMI) + static auto _v = get_config()->find("OMNITRACE_USE_ROCM_SMI"); + return static_cast&>(*_v->second).get(); #else static auto _v = false; return _v; @@ -443,10 +648,9 @@ get_use_sampling() static auto _v = get_config()->find("OMNITRACE_USE_SAMPLING"); return static_cast&>(*_v->second).get(); #else + OMNITRACE_THROW( + "Error! sampling was enabled but omnitrace was not built with libunwind support"); static bool _v = false; - if(_v) - throw std::runtime_error("Error! sampling was enabled but omnitrace was not " - "built with libunwind support"); return _v; #endif } @@ -472,6 +676,13 @@ get_use_critical_trace() return static_cast&>(*_v->second).get(); } +bool +get_use_kokkosp() +{ + static auto _v = get_config()->find("OMNITRACE_USE_KOKKOSP"); + return static_cast&>(*_v->second).get(); +} + bool get_critical_trace_debug() { @@ -634,6 +845,20 @@ get_critical_trace_count() return static_cast&>(*_v->second).get(); } +double& +get_rocm_smi_freq() +{ + static auto _v = get_config()->find("OMNITRACE_ROCM_SMI_FREQ"); + return static_cast&>(*_v->second).get(); +} + +std::string +get_rocm_smi_devices() +{ + static auto _v = get_config()->find("OMNITRACE_ROCM_SMI_DEVICES"); + return static_cast&>(*_v->second).get(); +} + bool get_debug_tid() { @@ -665,6 +890,7 @@ get_debug_pid() _vlist.count(dmp::rank()) > 0; return _v; } +} // namespace config State& get_state() @@ -685,8 +911,7 @@ get_cpu_cid_stack(int64_t _tid) { struct omnitrace_cpu_cid_stack {}; - using thread_data_t = - omnitrace_thread_data, omnitrace_cpu_cid_stack>; + using thread_data_t = thread_data, omnitrace_cpu_cid_stack>; static auto& _v = thread_data_t::instances(); static thread_local auto _v_check = [_tid]() { thread_data_t::construct((_tid > 0) ? *thread_data_t::instances().at(0) diff --git a/projects/rocprofiler-systems/src/library/critical_trace.cpp b/projects/rocprofiler-systems/src/library/critical_trace.cpp index 61bb97a60e..35dfa24050 100644 --- a/projects/rocprofiler-systems/src/library/critical_trace.cpp +++ b/projects/rocprofiler-systems/src/library/critical_trace.cpp @@ -29,6 +29,7 @@ #include #include +#include #include #include #include @@ -38,6 +39,7 @@ #include #include +#include #include #include #include @@ -176,7 +178,8 @@ entry::operator+=(const entry& rhs) } else { - OMNITRACE_PRINT( + OMNITRACE_CONDITIONAL_PRINT( + get_verbose() > 1, "Warning! Incorrect phase. entry::operator+=(entry) is only valid for " "Phase::BEGIN += Phase::END\n"); } @@ -427,46 +430,80 @@ call_chain::query(FuncT&& _func) const template <> void -call_chain::generate_perfetto(std::set&) const +call_chain::generate_perfetto(std::set&, bool) const {} template <> void -call_chain::generate_perfetto(std::set& _used) const +call_chain::generate_perfetto(std::set& _used, bool _basic) const { static std::set _static_strings{}; static std::mutex _static_mutex{}; for(const auto& itr : *this) { - if(!_used.emplace(itr).second) continue; - std::string _name = tim::demangle(tim::get_hash_identifier(itr.hash)); - _static_mutex.lock(); - auto sitr = _static_strings.emplace(_name); - _static_mutex.unlock(); - TRACE_EVENT_BEGIN("host-critical-trace", - perfetto::StaticString{ sitr.first->c_str() }, - static_cast(itr.begin_ns)); - TRACE_EVENT_END("host-critical-trace", static_cast(itr.end_ns)); + if(_basic) + { + if(itr.device == Device::CPU) + { + TRACE_EVENT_BEGIN("device-critical-trace", "CPU", + static_cast(itr.begin_ns)); + } + else if(itr.device == Device::GPU) + { + TRACE_EVENT_BEGIN("device-critical-trace", "GPU", + static_cast(itr.begin_ns)); + } + TRACE_EVENT_END("device-critical-trace", static_cast(itr.end_ns)); + } + else + { + if(!_used.emplace(itr).second) continue; + std::string _name = tim::demangle(tim::get_hash_identifier(itr.hash)); + _static_mutex.lock(); + auto sitr = _static_strings.emplace(_name); + _static_mutex.unlock(); + TRACE_EVENT_BEGIN("host-critical-trace", + perfetto::StaticString{ sitr.first->c_str() }, + static_cast(itr.begin_ns)); + TRACE_EVENT_END("host-critical-trace", static_cast(itr.end_ns)); + } } } template <> void -call_chain::generate_perfetto(std::set& _used) const +call_chain::generate_perfetto(std::set& _used, bool _basic) const { static std::set _static_strings{}; static std::mutex _static_mutex{}; for(const auto& itr : *this) { - if(!_used.emplace(itr).second) continue; - std::string _name = tim::demangle(tim::get_hash_identifier(itr.hash)); - _static_mutex.lock(); - auto sitr = _static_strings.emplace(_name); - _static_mutex.unlock(); - TRACE_EVENT_BEGIN("device-critical-trace", - perfetto::StaticString{ sitr.first->c_str() }, - static_cast(itr.begin_ns)); - TRACE_EVENT_END("device-critical-trace", static_cast(itr.end_ns)); + if(_basic) + { + if(itr.device == Device::CPU) + { + TRACE_EVENT_BEGIN("device-critical-trace", "CPU", + static_cast(itr.begin_ns)); + } + else if(itr.device == Device::GPU) + { + TRACE_EVENT_BEGIN("device-critical-trace", "GPU", + static_cast(itr.begin_ns)); + } + TRACE_EVENT_END("device-critical-trace", static_cast(itr.end_ns)); + } + else + { + if(!_used.emplace(itr).second) continue; + std::string _name = tim::demangle(tim::get_hash_identifier(itr.hash)); + _static_mutex.lock(); + auto sitr = _static_strings.emplace(_name); + _static_mutex.unlock(); + TRACE_EVENT_BEGIN("device-critical-trace", + perfetto::StaticString{ sitr.first->c_str() }, + static_cast(itr.begin_ns)); + TRACE_EVENT_END("device-critical-trace", static_cast(itr.end_ns)); + } } } @@ -485,7 +522,7 @@ get_update_frequency() std::unique_ptr& get(int64_t _tid) { - static auto& _v = omnitrace_thread_data::instances(); + static auto& _v = thread_data::instances(); static thread_local auto _once = [_tid]() { if(!_v.at(0)) _v.at(0) = std::make_unique(); if(!_v.at(_tid)) _v.at(_tid) = std::make_unique(); @@ -515,10 +552,10 @@ size_t add_hash_id(const std::string& _label) { using critical_trace_hash_data = - omnitrace_thread_data; + thread_data; auto _hash = tim::hash::add_hash_id(_label); - if(get_use_critical_trace()) + if(get_use_critical_trace() || get_use_rocm_smi()) { critical_trace_hash_data::construct(); critical_trace_hash_data::instance()->emplace(_label); @@ -529,7 +566,7 @@ add_hash_id(const std::string& _label) void update(int64_t _tid) { - if(!get_use_critical_trace()) return; + if(!get_use_critical_trace() && !get_use_rocm_smi()) return; std::unique_lock _lk{ tasking::get_critical_trace_mutex(), std::defer_lock }; if(!_lk.owns_lock()) _lk.lock(); @@ -604,7 +641,8 @@ save_call_graph(const std::string& _fname, const std::string& _label, if(_msg) { if(_func.empty()) _func = __FUNCTION__; - OMNITRACE_PRINT("[%s] Outputting '%s'...\n", _func.c_str(), _fname.c_str()); + OMNITRACE_CONDITIONAL_PRINT(get_verbose() >= 0, "[%s] Outputting '%s'...\n", + _func.c_str(), _fname.c_str()); } ofs << oss.str() << std::endl; } @@ -648,7 +686,8 @@ save_critical_trace(const std::string& _fname, const std::string& _label, if(_msg) { if(_func.empty()) _func = __FUNCTION__; - OMNITRACE_PRINT("[%s] Outputting '%s'...\n", _func.c_str(), _fname.c_str()); + OMNITRACE_CONDITIONAL_PRINT(get_verbose() >= 0, "[%s] Outputting '%s'...\n", + _func.c_str(), _fname.c_str()); } std::stringstream oss{}; if(_cchain.size() > 1000) @@ -686,7 +725,8 @@ save_call_chain_text(const std::string& _fname, const call_chain& _call_chain, if(_msg) { if(_func.empty()) _func = __FUNCTION__; - OMNITRACE_PRINT("[%s] Outputting '%s'...\n", _func.c_str(), _fname.c_str()); + OMNITRACE_CONDITIONAL_PRINT(get_verbose() >= 0, "[%s] Outputting '%s'...\n", + _func.c_str(), _fname.c_str()); } ofs << _call_chain << "\n"; } @@ -730,7 +770,8 @@ save_call_chain_json(const std::string& _fname, const std::string& _label, if(_msg) { if(_func.empty()) _func = __FUNCTION__; - OMNITRACE_PRINT("[%s] Outputting '%s'...\n", _func.c_str(), _fname.c_str()); + OMNITRACE_CONDITIONAL_PRINT(get_verbose() >= 0, "[%s] Outputting '%s'...\n", + _func.c_str(), _fname.c_str()); } std::stringstream oss{}; if(_call_chain.size() > 100000) @@ -1117,27 +1158,33 @@ get_top(const std::vector& _chain, size_t _count) template void -generate_perfetto(const std::vector& _data) +generate_perfetto(const std::vector& _data, bool _basic = false) { OMNITRACE_CT_DEBUG("[%s]\n", __FUNCTION__); - auto _func = [&](size_t _beg, size_t _end) { + + auto _nrows = std::min(get_critical_trace_per_row(), _data.size()); + + // run in separate thread(s) so that it ends up in unique row + if(_nrows < 1) _nrows = _data.size(); + + auto _func = [&](size_t _idx, size_t _beg, size_t _end) { + if(_nrows != 1) + threading::set_thread_name(TIMEMORY_JOIN(" ", "CriticalPath", _idx).c_str()); + else + threading::set_thread_name("CritialPath"); // ensure all hash ids exist copy_hash_ids(); std::set _used{}; for(size_t i = _beg; i < _end; ++i) { if(i >= _data.size()) break; - _data.at(i).generate_perfetto(_used); + _data.at(i).generate_perfetto(_used, _basic); } }; - // run in separate thread(s) so that it ends up in unique row - auto _nrows = get_critical_trace_per_row(); - if(_nrows < 1) _nrows = _data.size(); - for(size_t i = 0; i < _data.size(); i += _nrows) { - std::thread{ _func, i, i + _nrows }.join(); + std::thread{ _func, i, i, i + _nrows }.join(); } } @@ -1225,6 +1272,8 @@ compute_critical_trace() _perf.reset().start(); + generate_perfetto({ complete_call_chain }, true); + OMNITRACE_CT_DEBUG("[%s] Finding sequences...\n", __FUNCTION__); // find the sequences std::vector _top{}; @@ -1280,8 +1329,7 @@ compute_critical_trace() _tg.join(); _tp.destroy_threadpool(); _computed = true; - - } catch(const std::exception& e) + } catch(std::exception& e) { OMNITRACE_PRINT("Thread exited '%s' with exception: %s\n", __FUNCTION__, e.what()); @@ -1295,5 +1343,36 @@ compute_critical_trace() OMNITRACE_PRINT("%s\n", _ct_msg.c_str()); } } // namespace + +std::vector> +get_entries(int64_t _ts, const std::function& _eval) +{ + copy_hash_ids(); + auto _func = [_eval, _ts](std::vector>* _targ, + size_t* _avail) { + copy_hash_ids(); + squash_critical_path(complete_call_chain); + *_avail = complete_call_chain.size(); + std::vector> _v{}; + std::sort(complete_call_chain.begin(), complete_call_chain.end()); + for(const auto& itr : complete_call_chain) + { + if(itr.phase != Phase::DELTA) continue; + if(itr.begin_ns <= _ts && itr.end_ns >= _ts) + { + if(_eval(itr)) _v.emplace_back(tim::get_hash_identifier(itr.hash), itr); + } + } + *_targ = _v; + }; + size_t _n = 0; + std::vector> _v{}; + tasking::get_critical_trace_task_group().exec(_func, &_v, &_n); + tasking::get_critical_trace_task_group().join(); + OMNITRACE_DEBUG("critical_trace::%s :: found %zu out of %zu entries at %li...\n", + __FUNCTION__, _v.size(), _n, _ts); + return _v; +} + } // namespace critical_trace } // namespace omnitrace diff --git a/projects/rocprofiler-systems/src/library/gpu.cpp b/projects/rocprofiler-systems/src/library/gpu.cpp index 16ca69dc34..8b03773461 100644 --- a/projects/rocprofiler-systems/src/library/gpu.cpp +++ b/projects/rocprofiler-systems/src/library/gpu.cpp @@ -22,11 +22,12 @@ #include "library/gpu.hpp" -#if defined(OMNITRACE_USE_HIP) +#if defined(OMNITRACE_USE_ROCM_SMI) +# include "library/components/rocm_smi.hpp" +#elif defined(OMNITRACE_USE_HIP) # if !defined(TIMEMORY_USE_HIP) # define TIMEMORY_USE_HIP 1 # endif - # include "timemory/components/hip/backends.hpp" #endif @@ -34,18 +35,18 @@ namespace omnitrace { namespace gpu { -#if defined(OMNITRACE_USE_HIP) int device_count() { +#if defined(OMNITRACE_USE_ROCM_SMI) + // store as static since calls after rsmi_shutdown will return zero + static auto _v = rocm_smi::device_count(); + return _v; +#elif defined(OMNITRACE_USE_HIP) return ::tim::hip::device_count(); -} #else -int -device_count() -{ return 0; -} #endif +} } // namespace gpu } // namespace omnitrace diff --git a/projects/rocprofiler-systems/src/library/kokkosp.cpp b/projects/rocprofiler-systems/src/library/kokkosp.cpp new file mode 100644 index 0000000000..e79e1a8231 --- /dev/null +++ b/projects/rocprofiler-systems/src/library/kokkosp.cpp @@ -0,0 +1,326 @@ +// MIT License +// +// Copyright (c) 2020, The Regents of the University of California, +// through Lawrence Berkeley National Laboratory (subject to receipt of any +// required approvals from the U.S. Dept. of Energy). All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +// used by Kokkos decls +#if !defined(TIMEMORY_LIBRARY_SOURCE) +# define TIMEMORY_LIBRARY_SOURCE 1 +#endif + +#include "library/components/omnitrace.hpp" +#include "library/config.hpp" + +#include + +namespace kokkosp = tim::kokkosp; + +//--------------------------------------------------------------------------------------// + +namespace tim +{ +template <> +inline auto +invoke_preinit(long) +{ + kokkosp::memory_tracker::label() = "kokkos_memory"; + kokkosp::memory_tracker::description() = "Kokkos Memory tracker"; +} +} // namespace tim + +//--------------------------------------------------------------------------------------// + +namespace +{ +std::string kokkos_banner = + "#---------------------------------------------------------------------------#"; + +//--------------------------------------------------------------------------------------// + +bool enable_kernel_logger = false; + +inline void +add_kernel_logger() +{ + static bool _first = true; + if(!_first) return; + _first = false; + using strvec_t = std::vector; + + tim::settings::instance()->insert( + std::string{ "OMNITRACE_KOKKOS_KERNEL_LOGGER" }, std::string{}, + std::string{ "Enables kernel logging" }, enable_kernel_logger, + strvec_t({ "--omnitrace-kokkos-kernel-logger" })); +} + +inline void +setup_kernel_logger() +{ + if(tim::settings::debug() || tim::settings::verbose() > 3 || enable_kernel_logger) + { + kokkosp::logger_t::get_initializer() = [](kokkosp::logger_t& _obj) { + _obj.initialize(); + }; + } +} + +} // namespace + +//--------------------------------------------------------------------------------------// + +extern "C" +{ + void kokkosp_print_help(char*) {} + + void kokkosp_parse_args(int, char**) {} + + void kokkosp_declare_metadata(const char* key, const char* value) + { + tim::manager::add_metadata(key, value); + } + + void kokkosp_init_library(const int loadSeq, const uint64_t interfaceVer, + const uint32_t devInfoCount, void* deviceInfo) + { + add_kernel_logger(); + + tim::consume_parameters(devInfoCount, deviceInfo); + + printf("%s\n", kokkos_banner.c_str()); + printf("# KokkosP: omnitrace connector (sequence is %d, version: %llu)\n", + loadSeq, (unsigned long long) interfaceVer); + printf("%s\n", kokkos_banner.c_str()); + + setup_kernel_logger(); + + tim::trait::runtime_enabled::set( + omnitrace::config::get_use_timemory()); + } + + void kokkosp_finalize_library() + { + printf("%s\n", kokkos_banner.c_str()); + printf("# KokkosP: Finalization of omnitrace connector. Complete.\n"); + printf("%s\n", kokkos_banner.c_str()); + + kokkosp::cleanup(); + } + + //----------------------------------------------------------------------------------// + + void kokkosp_begin_parallel_for(const char* name, uint32_t devid, uint64_t* kernid) + { + auto pname = + (devid > std::numeric_limits::max()) // junk device number + ? TIMEMORY_JOIN(" ", "[kokkos]", name) + : TIMEMORY_JOIN(" ", TIMEMORY_JOIN("", "[kokkos][dev", devid, ']'), name); + *kernid = kokkosp::get_unique_id(); + kokkosp::logger_t{}.mark(1, __FUNCTION__, name, *kernid); + kokkosp::create_profiler(pname, *kernid); + kokkosp::start_profiler(*kernid); + } + + void kokkosp_end_parallel_for(uint64_t kernid) + { + kokkosp::logger_t{}.mark(-1, __FUNCTION__, kernid); + kokkosp::stop_profiler(kernid); + kokkosp::destroy_profiler(kernid); + } + + //----------------------------------------------------------------------------------// + + void kokkosp_begin_parallel_reduce(const char* name, uint32_t devid, uint64_t* kernid) + { + auto pname = + (devid > std::numeric_limits::max()) // junk device number + ? TIMEMORY_JOIN(" ", "[kokkos]", name) + : TIMEMORY_JOIN(" ", TIMEMORY_JOIN("", "[kokkos][dev", devid, ']'), name); + *kernid = kokkosp::get_unique_id(); + kokkosp::logger_t{}.mark(1, __FUNCTION__, name, *kernid); + kokkosp::create_profiler(pname, *kernid); + kokkosp::start_profiler(*kernid); + } + + void kokkosp_end_parallel_reduce(uint64_t kernid) + { + kokkosp::logger_t{}.mark(-1, __FUNCTION__, kernid); + kokkosp::stop_profiler(kernid); + kokkosp::destroy_profiler(kernid); + } + + //----------------------------------------------------------------------------------// + + void kokkosp_begin_parallel_scan(const char* name, uint32_t devid, uint64_t* kernid) + { + auto pname = + (devid > std::numeric_limits::max()) // junk device number + ? TIMEMORY_JOIN(" ", "[kokkos]", name) + : TIMEMORY_JOIN(" ", TIMEMORY_JOIN("", "[kokkos][dev", devid, ']'), name); + *kernid = kokkosp::get_unique_id(); + kokkosp::logger_t{}.mark(1, __FUNCTION__, name, *kernid); + kokkosp::create_profiler(pname, *kernid); + kokkosp::start_profiler(*kernid); + } + + void kokkosp_end_parallel_scan(uint64_t kernid) + { + kokkosp::logger_t{}.mark(-1, __FUNCTION__, kernid); + kokkosp::stop_profiler(kernid); + kokkosp::destroy_profiler(kernid); + } + + //----------------------------------------------------------------------------------// + + void kokkosp_begin_fence(const char* name, uint32_t devid, uint64_t* kernid) + { + auto pname = + (devid > std::numeric_limits::max()) // junk device number + ? TIMEMORY_JOIN(" ", "[kokkos]", name) + : TIMEMORY_JOIN(" ", TIMEMORY_JOIN("", "[kokkos][dev", devid, ']'), name); + *kernid = kokkosp::get_unique_id(); + kokkosp::logger_t{}.mark(1, __FUNCTION__, name, *kernid); + kokkosp::create_profiler(pname, *kernid); + kokkosp::start_profiler(*kernid); + } + + void kokkosp_end_fence(uint64_t kernid) + { + kokkosp::logger_t{}.mark(-1, __FUNCTION__, kernid); + kokkosp::stop_profiler(kernid); + kokkosp::destroy_profiler(kernid); + } + + //----------------------------------------------------------------------------------// + + void kokkosp_push_profile_region(const char* name) + { + kokkosp::logger_t{}.mark(1, __FUNCTION__, name); + kokkosp::get_profiler_stack().push_back( + kokkosp::profiler_t(name)); + kokkosp::get_profiler_stack().back().start(); + } + + void kokkosp_pop_profile_region() + { + kokkosp::logger_t{}.mark(-1, __FUNCTION__); + if(kokkosp::get_profiler_stack().empty()) return; + kokkosp::get_profiler_stack().back().stop(); + kokkosp::get_profiler_stack().pop_back(); + } + + //----------------------------------------------------------------------------------// + + void kokkosp_create_profile_section(const char* name, uint32_t* secid) + { + *secid = kokkosp::get_unique_id(); + auto pname = TIMEMORY_JOIN(" ", "[kokkos]", name); + kokkosp::create_profiler(pname, *secid); + } + + void kokkosp_destroy_profile_section(uint32_t secid) + { + kokkosp::destroy_profiler(secid); + } + + //----------------------------------------------------------------------------------// + + void kokkosp_start_profile_section(uint32_t secid) + { + kokkosp::logger_t{}.mark(1, __FUNCTION__, secid); + kokkosp::start_profiler(secid); + } + + void kokkosp_stop_profile_section(uint32_t secid) + { + kokkosp::logger_t{}.mark(-1, __FUNCTION__, secid); + kokkosp::start_profiler(secid); + } + + //----------------------------------------------------------------------------------// + + void kokkosp_allocate_data(const SpaceHandle space, const char* label, + const void* const ptr, const uint64_t size) + { + kokkosp::logger_t{}.mark(0, __FUNCTION__, space.name, label, + TIMEMORY_JOIN("", '[', ptr, ']'), size); + kokkosp::profiler_alloc_t<>{ TIMEMORY_JOIN(" ", "[kokkos][allocate]", space.name, + label) } + .store(std::plus{}, size); + } + + void kokkosp_deallocate_data(const SpaceHandle space, const char* label, + const void* const ptr, const uint64_t size) + { + kokkosp::logger_t{}.mark(0, __FUNCTION__, space.name, label, + TIMEMORY_JOIN("", '[', ptr, ']'), size); + kokkosp::profiler_alloc_t<>{ TIMEMORY_JOIN(" ", "[kokkos][deallocate]", + space.name, label) } + .store(std::plus{}, size); + } + + //----------------------------------------------------------------------------------// + + void kokkosp_begin_deep_copy(SpaceHandle dst_handle, const char* dst_name, + const void* dst_ptr, SpaceHandle src_handle, + const char* src_name, const void* src_ptr, uint64_t size) + { + kokkosp::logger_t{}.mark(1, __FUNCTION__, dst_handle.name, dst_name, + TIMEMORY_JOIN("", '[', dst_ptr, ']'), src_handle.name, + src_name, TIMEMORY_JOIN("", '[', src_ptr, ']'), size); + + auto name = TIMEMORY_JOIN(" ", "[kokkos][deep_copy]", + TIMEMORY_JOIN('=', dst_handle.name, dst_name), + TIMEMORY_JOIN('=', src_handle.name, src_name)); + + auto& _data = kokkosp::get_profiler_stack(); + _data.emplace_back(name); + _data.back().audit(dst_handle, dst_name, dst_ptr, src_handle, src_name, src_ptr, + size); + _data.back().start(); + _data.back().store(std::plus{}, size); + } + + void kokkosp_end_deep_copy() + { + kokkosp::logger_t{}.mark(-1, __FUNCTION__); + auto& _data = kokkosp::get_profiler_stack(); + if(_data.empty()) return; + _data.back().store(std::minus{}, 0); + _data.back().stop(); + _data.pop_back(); + } + + //----------------------------------------------------------------------------------// + + void kokkosp_profile_event(const char* name) + { + kokkosp::profiler_t{}.mark(name); + } + + //----------------------------------------------------------------------------------// +} + +//--------------------------------------------------------------------------------------// + +TIMEMORY_INITIALIZE_STORAGE(kokkosp::memory_tracker) + +//--------------------------------------------------------------------------------------// diff --git a/projects/rocprofiler-systems/src/library/ptl.cpp b/projects/rocprofiler-systems/src/library/ptl.cpp index 97ffa1b3d2..0e5b9a414b 100644 --- a/projects/rocprofiler-systems/src/library/ptl.cpp +++ b/projects/rocprofiler-systems/src/library/ptl.cpp @@ -24,6 +24,7 @@ #include "library/config.hpp" #include "library/debug.hpp" #include "library/defines.hpp" +#include "library/sampling.hpp" #include #include @@ -39,10 +40,14 @@ auto _thread_pool_cfg = []() { _v.init = true; _v.use_affinity = false; _v.use_tbb = false; - _v.initializer = []() {}; - _v.finalizer = []() {}; - _v.priority = 5; - _v.pool_size = 1; + _v.initializer = []() { + sampling::block_signals(); + threading::set_thread_name( + TIMEMORY_JOIN('.', "ptl", PTL::Threading::GetThreadId()).c_str()); + }; + _v.finalizer = []() {}; + _v.priority = 5; + _v.pool_size = 1; return _v; }(); } diff --git a/projects/rocprofiler-systems/src/library/sampling.cpp b/projects/rocprofiler-systems/src/library/sampling.cpp index e54a595c66..77c869cb49 100644 --- a/projects/rocprofiler-systems/src/library/sampling.cpp +++ b/projects/rocprofiler-systems/src/library/sampling.cpp @@ -76,10 +76,10 @@ namespace omnitrace namespace sampling { using hw_counters = typename component::backtrace::hw_counters; -using signal_type_instances = omnitrace_thread_data, api::sampling>; -using backtrace_init_instances = omnitrace_thread_data; -using sampler_running_instances = omnitrace_thread_data; -using papi_vector_instances = omnitrace_thread_data; +using signal_type_instances = thread_data, api::sampling>; +using backtrace_init_instances = thread_data; +using sampler_running_instances = thread_data; +using papi_vector_instances = thread_data; namespace { @@ -134,6 +134,7 @@ get_signal_types(int64_t _tid) std::set setup() { + if(!get_use_sampling()) return std::set{}; return backtrace::configure(true); } diff --git a/projects/rocprofiler-systems/src/library/state.cpp b/projects/rocprofiler-systems/src/library/state.cpp new file mode 100644 index 0000000000..572ec09271 --- /dev/null +++ b/projects/rocprofiler-systems/src/library/state.cpp @@ -0,0 +1,53 @@ +// MIT License +// +// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include "library/state.hpp" + +#include + +namespace std +{ +std::string +to_string(omnitrace::State _v) +{ + switch(_v) + { + case omnitrace::State::DelayedInit: return "DelayedInit"; + case omnitrace::State::PreInit: return "PreInit"; + case omnitrace::State::Init: return "Init"; + case omnitrace::State::Active: return "Active"; + case omnitrace::State::Finalized: return "Finalized"; + } + return {}; +} + +std::string +to_string(omnitrace::Mode _v) +{ + switch(_v) + { + case omnitrace::Mode::Trace: return "Trace"; + case omnitrace::Mode::Sampling: return "Sampling"; + } + return {}; +} +} // namespace std diff --git a/projects/rocprofiler-systems/src/omnitrace.cpp b/projects/rocprofiler-systems/src/omnitrace.cpp index 838f230355..7591a2cc8d 100644 --- a/projects/rocprofiler-systems/src/omnitrace.cpp +++ b/projects/rocprofiler-systems/src/omnitrace.cpp @@ -23,52 +23,55 @@ #include "omnitrace.hpp" #include +#include #include +#include #include #include #include #include +#include +#include bool debug_print = false; int verbose_level = tim::get_env("TIMEMORY_RUN_VERBOSE", 0); -static bool binary_rewrite = false; -static bool is_attached = false; -static bool loop_level_instr = false; -static bool werror = false; -static bool stl_func_instr = false; -static bool use_mpi = false; -static bool is_static_exe = false; -static bool is_driver = false; -static bool allow_overlapping = false; -static bool instr_dynamic_callsites = false; -static bool instr_traps = false; -static bool instr_loop_traps = false; -static size_t batch_size = 50; -static strset_t extra_libs = {}; -static size_t min_address_range = (1 << 8); // 256 -static size_t min_loop_address_range = (1 << 8); // 256 -static std::vector> hash_ids = {}; -static std::map use_stubs = {}; -static std::map beg_stubs = {}; -static std::map end_stubs = {}; -static strvec_t init_stub_names = {}; -static strvec_t fini_stub_names = {}; -static strset_t used_stub_names = {}; -static std::vector env_variables = {}; -static std::map beg_expr = {}; -static std::map end_expr = {}; -static const auto npos_v = string_t::npos; -static string_t instr_mode = "trace"; -static string_t instr_push_func = "omnitrace_push_trace"; -static string_t instr_pop_func = "omnitrace_pop_trace"; -static string_t instr_push_hash = "omnitrace_push_trace_hash"; -static string_t instr_pop_hash = "omnitrace_pop_trace_hash"; -static string_t print_instrumented = {}; -static string_t print_available = {}; -static string_t print_overlapping = {}; -static std::string modfunc_dump_dir = {}; -static auto regex_opts = std::regex_constants::egrep | std::regex_constants::optimize; +namespace +{ +bool binary_rewrite = false; +bool is_attached = false; +bool loop_level_instr = false; +bool werror = false; +bool stl_func_instr = false; +bool use_mpi = false; +bool is_static_exe = false; +bool is_driver = false; +bool allow_overlapping = false; +bool instr_dynamic_callsites = false; +bool instr_traps = false; +bool instr_loop_traps = false; +size_t batch_size = 50; +strset_t extra_libs = {}; +size_t min_address_range = (1 << 8); // 256 +size_t min_loop_address_range = (1 << 8); // 256 +std::vector> hash_ids = {}; +std::map use_stubs = {}; +std::map beg_stubs = {}; +std::map end_stubs = {}; +strvec_t init_stub_names = {}; +strvec_t fini_stub_names = {}; +strset_t used_stub_names = {}; +std::vector env_variables = {}; +std::map beg_expr = {}; +std::map end_expr = {}; +const auto npos_v = string_t::npos; +string_t instr_mode = "trace"; +string_t print_instrumented = {}; +string_t print_available = {}; +string_t print_overlapping = {}; +std::string modfunc_dump_dir = {}; +auto regex_opts = std::regex_constants::egrep | std::regex_constants::optimize; +} // namespace std::string get_absolute_exe_filepath(std::string exe_name); @@ -132,10 +135,10 @@ main(int argc, char** argv) bpatch->setLivenessAnalysis(false); bpatch->setBaseTrampDeletion(false); bpatch->setTrampRecursive(false); - bpatch->setMergeTramp(false); + bpatch->setMergeTramp(true); std::set dyninst_defs = { "TypeChecking", "SaveFPR", "DelayedParsing", - "InstrStackFrames" }; + "MergeTramp" }; int _argc = argc; int _cmdc = 0; @@ -598,7 +601,8 @@ main(int argc, char** argv) // Helper function for adding regex expressions // auto add_regex = [](auto& regex_array, const string_t& regex_expr) { - if(!regex_expr.empty()) regex_array.push_back(std::regex(regex_expr, regex_opts)); + if(!regex_expr.empty()) + regex_array.emplace_back(std::regex(regex_expr, regex_opts)); }; add_regex(func_include, tim::get_env("OMNITRACE_REGEX_INCLUDE", "")); @@ -638,7 +642,7 @@ main(int argc, char** argv) // //----------------------------------------------------------------------------------// - int dyninst_verb = 1; + int dyninst_verb = 2; if(parser.exists("dyninst-options")) { dyninst_defs = parser.get>("dyninst-options"); @@ -704,6 +708,26 @@ main(int argc, char** argv) // //----------------------------------------------------------------------------------// + // for runtime instrumentation, we need to set this before the process gets created + if(!binary_rewrite) + { + tim::set_env("HSA_ENABLE_INTERRUPT", "0", 0); + if(_pid >= 0) + { + verbprintf(-10, "#-------------------------------------------------------" + "-------------------------------------------#\n"); + verbprintf(-10, "\n"); + verbprintf(-10, "WARNING! Sampling may result in ioctl() deadlock within " + "the ROCR runtime.\n"); + verbprintf(-10, + "To avoid this, set HSA_ENABLE_INTERRUPT=0 in the environment " + "before starting your ROCm/HIP application\n"); + verbprintf(-10, "\n"); + verbprintf(-10, "#-------------------------------------------------------" + "-------------------------------------------#\n"); + } + } + addr_space = omnitrace_get_address_space(bpatch, _cmdc, _cmdv, binary_rewrite, _pid, mutname); @@ -750,7 +774,7 @@ main(int argc, char** argv) if(app_modules && !app_modules->empty()) { modules = *app_modules; - for(auto* itr : *app_modules) + for(auto* itr : modules) { auto* procedures = itr->getProcedures(); if(procedures) @@ -774,7 +798,7 @@ main(int argc, char** argv) if(app_functions && !app_functions->empty()) { functions = *app_functions; - for(auto* itr : *app_functions) + for(auto* itr : functions) { module_t* mod = itr->getModule(); if(mod && itr->isInstrumentable()) @@ -792,9 +816,9 @@ main(int argc, char** argv) } verbprintf(0, "Module size before loading instrumentation library: %lu\n", - (long unsigned) app_modules->size()); + (long unsigned) modules.size()); - if(debug_print || verbose_level > 1) + if(debug_print || verbose_level > 2) { module_function::reset_width(); for(const auto& itr : available_module_functions) @@ -931,21 +955,18 @@ main(int argc, char** argv) verbprintf(0, "Finding functions in image...\n"); - auto* entr_trace = find_function(app_image, instr_push_func.c_str()); - auto* exit_trace = find_function(app_image, instr_pop_func.c_str()); - auto* entr_hash = find_function(app_image, instr_push_hash.c_str()); - auto* exit_hash = find_function(app_image, instr_pop_hash.c_str()); - auto* init_func = find_function(app_image, "omnitrace_trace_init"); - auto* fini_func = find_function(app_image, "omnitrace_trace_finalize"); - auto* env_func = find_function(app_image, "omnitrace_trace_set_env"); - auto* mpi_func = find_function(app_image, "omnitrace_trace_set_mpi"); + auto* entr_trace = find_function(app_image, "omnitrace_push_trace"); + auto* exit_trace = find_function(app_image, "omnitrace_pop_trace"); + auto* entr_hash = find_function(app_image, "omnitrace_push_trace_hash"); + auto* exit_hash = find_function(app_image, "omnitrace_pop_trace_hash"); + auto* init_func = find_function(app_image, "omnitrace_init"); + auto* fini_func = find_function(app_image, "omnitrace_finalize"); + auto* env_func = find_function(app_image, "omnitrace_set_env"); + auto* mpi_func = find_function(app_image, "omnitrace_set_mpi"); auto* hash_func = find_function(app_image, "omnitrace_add_hash_id"); if(!main_func && main_fname == "main") main_func = find_function(app_image, "_main"); - verbprintf(0, "Instrumenting with '%s' and '%s'...\n", instr_push_func.c_str(), - instr_pop_func.c_str()); - if(mpi_init_func && mpi_fini_func) use_mpi = true; bool use_mpip = false; @@ -1085,10 +1106,10 @@ main(int argc, char** argv) using pair_t = std::pair; for(const auto& itr : - { pair_t(main_func, main_fname), pair_t(entr_trace, instr_push_func), - pair_t(exit_trace, instr_pop_func), pair_t(init_func, "omnitrace_trace_init"), - pair_t(fini_func, "omnitrace_trace_finalize"), - pair_t(env_func, "omnitrace_trace_set_env") }) + { pair_t(main_func, main_fname), pair_t(entr_trace, "omnitrace_push_trace"), + pair_t(exit_trace, "omnitrace_pop_trace"), pair_t(init_func, "omnitrace_init"), + pair_t(fini_func, "omnitrace_finalize"), + pair_t(env_func, "omnitrace_set_env") }) { if(itr.first == main_func && !is_driver) continue; if(!itr.first) @@ -1182,7 +1203,7 @@ main(int argc, char** argv) verbprintf(2, "Getting call expressions... "); auto main_call_args = omnitrace_call_expr(main_sign.get()); - auto init_call_args = omnitrace_call_expr(default_components, binary_rewrite, cmdv0); + auto init_call_args = omnitrace_call_expr(instr_mode, binary_rewrite, cmdv0); auto fini_call_args = omnitrace_call_expr(); auto umpi_call_args = omnitrace_call_expr(use_mpi, is_attached); auto none_call_args = omnitrace_call_expr(); @@ -1208,11 +1229,19 @@ main(int argc, char** argv) } } + std::string _libname = {}; + for(auto&& itr : sharedlibname) + _libname = get_absolute_lib_filepath(itr); + if(_libname.empty()) _libname = "libomnitrace.so"; + auto env_vars = parser.get("env"); env_vars.emplace_back(TIMEMORY_JOIN('=', "OMNITRACE_MODE", instr_mode)); + env_vars.emplace_back(TIMEMORY_JOIN('=', "HSA_ENABLE_INTERRUPT", "0")); + env_vars.emplace_back(TIMEMORY_JOIN('=', "HSA_TOOLS_LIB", _libname)); env_vars.emplace_back(TIMEMORY_JOIN('=', "OMNITRACE_MPI_INIT", "OFF")); env_vars.emplace_back(TIMEMORY_JOIN('=', "OMNITRACE_MPI_FINALIZE", "OFF")); - env_vars.emplace_back(TIMEMORY_JOIN('=', "OMNITRACE_COMPONENTS", default_components)); + env_vars.emplace_back( + TIMEMORY_JOIN('=', "OMNITRACE_TIMEMORY_COMPONENTS", default_components)); env_vars.emplace_back( TIMEMORY_JOIN('=', "OMNITRACE_USE_MPIP", (binary_rewrite && use_mpi && use_mpip) ? "ON" : "OFF")); @@ -1229,7 +1258,7 @@ main(int argc, char** argv) } tim::set_env(p.at(0), p.at(1)); auto _expr = omnitrace_call_expr(p.at(0), p.at(1)); - env_variables.push_back(_expr.get(env_func)); + env_variables.emplace_back(_expr.get(env_func)); } //----------------------------------------------------------------------------------// @@ -1240,7 +1269,7 @@ main(int argc, char** argv) for(const auto& itr : env_variables) { - if(itr) init_names.push_back(itr.get()); + if(itr) init_names.emplace_back(itr.get()); } for(const auto& itr : beg_expr) @@ -1248,7 +1277,7 @@ main(int argc, char** argv) if(itr.second) { verbprintf(1, "+ Adding %s instrumentation...\n", itr.first.c_str()); - init_names.push_back(itr.second.get()); + init_names.emplace_back(itr.second.get()); } else { @@ -1256,55 +1285,35 @@ main(int argc, char** argv) } } - if(use_mpi && umpi_call) init_names.push_back(umpi_call.get()); - - if(init_call && binary_rewrite) init_names.push_back(init_call.get()); + if(umpi_call) init_names.emplace_back(umpi_call.get()); + if(init_call) init_names.emplace_back(init_call.get()); if(binary_rewrite) { - if(mpi_init_func && mpi_fini_func) - { - verbprintf(2, "Patching MPI init functions\n"); - if(init_call) - insert_instr(addr_space, mpi_init_func, init_call, BPatch_exit, nullptr, - nullptr); - if(fini_call) - insert_instr(addr_space, mpi_fini_func, fini_call, BPatch_entry, nullptr, - nullptr); - } - else - { - verbprintf(2, "Adding main begin and end snippets...\n"); - init_names.push_back(main_beg_call.get()); - fini_names.push_back(main_end_call.get()); - } + verbprintf(2, "Adding main begin and end snippets...\n"); + if(main_beg_call) init_names.emplace_back(main_beg_call.get()); + if(main_end_call) fini_names.emplace_back(main_end_call.get()); } else if(app_thread) { verbprintf(2, "Patching main function\n"); - if(init_call) - insert_instr(addr_space, main_func, init_call, BPatch_entry, nullptr, - nullptr); - if(!use_mpi) - { + if(main_beg_call) insert_instr(addr_space, main_func, main_beg_call, BPatch_entry, nullptr, nullptr); + if(main_end_call) insert_instr(addr_space, main_func, main_end_call, BPatch_exit, nullptr, nullptr); - } - if(fini_call) - insert_instr(addr_space, main_func, fini_call, BPatch_exit, nullptr, nullptr); } else { verbprintf(0, "No binary_rewrite and no app_thread!...\n"); } - if(fini_call) fini_names.push_back(fini_call.get()); + if(fini_call) fini_names.emplace_back(fini_call.get()); for(const auto& itr : end_expr) { - if(itr.second) fini_names.push_back(itr.second.get()); + if(itr.second) fini_names.emplace_back(itr.second.get()); } //----------------------------------------------------------------------------------// @@ -1316,7 +1325,7 @@ main(int argc, char** argv) //----------------------------------------------------------------------------------// std::vector> instr_procedure_functions; auto instr_procedures = [&](const procedure_vec_t& procedures) { - verbprintf(2, "Instrumenting %lu procedures...\n", + verbprintf(3, "Instrumenting %lu procedures...\n", (unsigned long) procedures.size()); for(auto* itr : procedures) { @@ -1356,7 +1365,7 @@ main(int argc, char** argv) if(name.get().empty()) { - verbprintf(1, "Skipping function [empty name]: %s\n", fname); + verbprintf(2, "Skipping function [empty name]: %s\n", fname); continue; } @@ -1366,7 +1375,7 @@ main(int argc, char** argv) if(is_static_exe && has_debug_info && string_t{ fname } == "_fini" && string_t{ modname } == "DEFAULT_MODULE") { - verbprintf(1, "Skipping function [DEFAULT_MODULE]: %s\n", fname); + verbprintf(2, "Skipping function [DEFAULT_MODULE]: %s\n", fname); continue; } @@ -1376,7 +1385,7 @@ main(int argc, char** argv) overlapping_module_functions.find(module_function{ mod, itr }) != overlapping_module_functions.end()) { - verbprintf(1, "Skipping function [overlapping]: %s / %s\n", + verbprintf(2, "Skipping function [overlapping]: %s / %s\n", name.m_name.c_str(), name.get().c_str()); continue; } @@ -1402,8 +1411,33 @@ main(int argc, char** argv) ? std::max(_loop_entries->size(), basic_loop.size()) : basic_loop.size(); auto _has_loop_entries = (_num_loop_entries > 0); + auto _skip_range = + (_has_loop_entries) ? false : (_address_range < min_address_range); + auto _skip_loop_range = + (_has_loop_entries) ? (_address_range < min_loop_address_range) : false; - if(_address_range < min_address_range && !_has_loop_entries && !_force_instr) + if(_force_instr && _skip_range) + { + verbprintf( + 1, + "Instrumenting function [dynamic-callsite]: %s / %s despite not " + "satisfy minimum address range (address range = %lu, minimum " + "= %lu) because contains dynamic callsites\n", + name.m_name.c_str(), name.get().c_str(), + (unsigned long) _address_range, (unsigned long) min_address_range); + } + else if(_force_instr && _skip_loop_range) + { + verbprintf( + 1, + "Instrumenting function [dynamic-callsite]: %s / %s despite not " + "satisfy minimum loop address range (address range = %lu, minimum " + "= %lu) because contains dynamic callsites\n", + name.m_name.c_str(), name.get().c_str(), + (unsigned long) _address_range, + (unsigned long) min_loop_address_range); + } + else if(_skip_range) { verbprintf(1, "Skipping function [min-address-range]: %s / %s (address " @@ -1413,8 +1447,7 @@ main(int argc, char** argv) (unsigned long) min_address_range); continue; } - else if(_address_range < min_loop_address_range && _has_loop_entries && - !_force_instr) + else if(_skip_loop_range) { verbprintf(1, "Skipping function [min-loop-address-range]: %s / %s (address " @@ -1424,16 +1457,6 @@ main(int argc, char** argv) (unsigned long) min_loop_address_range); continue; } - else if(_force_instr) - { - verbprintf(1, - "Enabling function [dynamic-callsite]: %s / %s despite not " - "satisfy minimum address range (address range = %lu, minimum " - "= %lu) because contains dynamic callsites\n", - name.m_name.c_str(), name.get().c_str(), - (unsigned long) _address_range, - (unsigned long) min_address_range); - } bool _entr_success = query_instr(itr, BPatch_entry, nullptr, nullptr, instr_traps); @@ -1562,7 +1585,7 @@ main(int argc, char** argv) continue; } - verbprintf(1, "Parsing module: %s\n", modname); + verbprintf(3, "Parsing module: %s\n", modname); bpvector_t* p = m->getProcedures(); if(!p) continue; @@ -1590,7 +1613,7 @@ main(int argc, char** argv) // //----------------------------------------------------------------------------------// - if(is_attached) + if(app_thread) { assert(app_thread != nullptr); verbprintf(1, "Executing initial snippets...\n"); @@ -1827,58 +1850,52 @@ main(int argc, char** argv) { verbprintf(0, "Executing...\n"); - // bpatch->setDebugParsing(false); - // bpatch->setTypeChecking(false); - // bpatch->setDelayedParsing(true); - // bpatch->setInstrStackFrames(true); - // bpatch->setLivenessAnalysis(false); - // addr_space->beginInsertionSet(); - - verbprintf(4, "Registering fork callbacks...\n"); - auto _prefork = bpatch->registerPreForkCallback(&omnitrace_fork_callback); - auto _postfork = bpatch->registerPostForkCallback(&omnitrace_fork_callback); - - auto _wait_exec = [&]() { - while(!app_thread->isTerminated()) + if(!app_thread->isTerminated()) + { + app_thread->detach(true); + pid_t cpid = app_thread->getPid(); + pid_t w; + int status = 0; + do { - verbprintf(3, "Continuing execution...\n"); - app_thread->continueExecution(); - verbprintf(4, "Process is not terminated...\n"); - bpatch->waitForStatusChange(); - std::this_thread::sleep_for(std::chrono::milliseconds{ 100 }); - verbprintf(4, "Process status change...\n"); - if(app_thread->isStopped()) + w = waitpid(cpid, &status, WUNTRACED); + if(w == -1) { - verbprintf(4, "Process is stopped, continuing execution...\n"); - if(!app_thread->continueExecution()) - { - fprintf(stderr, "continueExecution failed\n"); - exit(EXIT_FAILURE); - } + perror("waitpid"); + exit(EXIT_FAILURE); } + + if(WIFEXITED(status)) + { + code = WEXITSTATUS(status); + } + else if(WIFSIGNALED(status)) + { + code = WTERMSIG(status); + } + else if(WIFSTOPPED(status)) + { + code = WSTOPSIG(status); + } + else if(WIFCONTINUED(status)) + { + code = WIFCONTINUED(status); + } + } while(!WIFEXITED(status) && !WIFSIGNALED(status)); + } + else + { + if(app_thread->terminationStatus() == ExitedNormally) + { + if(app_thread->isTerminated()) verbprintf(0, "End of omnitrace\n"); } - }; - - verbprintf(4, "Entering wait for status change mode...\n"); - _wait_exec(); - - if(app_thread->terminationStatus() == ExitedNormally) - { - if(app_thread->isTerminated()) - printf("\nEnd of omnitrace\n"); - else - _wait_exec(); + else if(app_thread->terminationStatus() == ExitedViaSignal) + { + auto sign = app_thread->getExitSignal(); + fprintf(stderr, "\nApplication exited with signal: %i\n", int(sign)); + } + code = app_thread->getExitCode(); } - else if(app_thread->terminationStatus() == ExitedViaSignal) - { - auto sign = app_thread->getExitSignal(); - fprintf(stderr, "\nApplication exited with signal: %i\n", int(sign)); - } - - // addr_space->finalizeInsertionSet(false, nullptr); - - code = app_thread->getExitCode(); - consume_parameters(_prefork, _postfork); } // cleanup @@ -1888,6 +1905,9 @@ main(int argc, char** argv) for(int i = 0; i < _cmdc; ++i) delete[] _cmdv[i]; delete[] _cmdv; + + verbprintf(0, "End of omnitrace\n"); + verbprintf(1, "Exit code: %i\n", code); return code; } @@ -1943,7 +1963,7 @@ instrument_module(const string_t& file_name) static std::regex corelib_regex( "^lib(c|z|rt|dl|dw|util|zstd|elf|pthread|open[\\-]rte|open[\\-]pal|" "hwloc|numa|event|udev|dyninstAPI_RT|gcc_s|tcmalloc|profiler|tbbmalloc|" - "tbbmalloc_proxy)(-|\\.)", + "tbbmalloc_proxy|event_pthreads|ltdl)(-|\\.)", regex_opts); // these are all due to TAU static std::regex prefix_regex( @@ -2049,11 +2069,13 @@ instrument_entity(const string_t& function_name) regex_opts); static std::regex exclude_cxx("(std::_Sp_counted_base|std::use_facet)", regex_opts); static std::regex leading( - "^(_|\\.|frame_dummy|\\(|targ|new|delete|operator new|operator " - "delete|std::allocat|" - "nvtx|gcov|main\\.cold|TAU|tau|Tau|dyn|RT|dl|sys|pthread|posix|clone|virtual " - "thunk|non-virtual thunk|transaction " - "clone|RtsLayer|DYNINST|PthreadLayer|threaded_func|targ8|PMPI)", + "^(_|\\.|frame_dummy|\\(|targ|new|delete|operator new|operator delete|" + "std::allocat|nvtx|gcov|main\\.cold|TAU|tau|Tau|dyn|RT|" + "sys|pthread|posix|clone|" + "virtual thunk|non-virtual thunk|transaction clone|" + "RtsLayer|DYNINST|PthreadLayer|threaded_func|PMPI|" + "Kokkos::Impl::|Kokkos::Experimental::Impl::|Kokkos::impl_|" + "Kokkos::[A-Za-z]+::impl_|Kokkos::Tools::|Kokkos::Profiling::)", regex_opts); static std::regex trailing("(\\.part\\.[0-9]+|\\.constprop\\.[0-9]+|\\.|\\.[0-9]+)$", regex_opts); @@ -2328,7 +2350,8 @@ get_absolute_lib_filepath(std::string lib_name) std::regex_match(lib_name, std::regex("^[A-Za-z0-9].*")))) { auto _lib_orig = lib_name; - auto _paths = tim::delimit(tim::get_env("LD_LIBRARY_PATH", ""), ":"); + auto _paths = tim::delimit( + std::string{ ".:" } + tim::get_env("LD_LIBRARY_PATH", ""), ":"); for(auto& pitr : _paths) { if(file_exists(TIMEMORY_JOIN('/', pitr, lib_name))) diff --git a/projects/rocprofiler-systems/src/omnitrace/details.cpp b/projects/rocprofiler-systems/src/omnitrace/details.cpp index 6dcac4ed95..1548371f4f 100644 --- a/projects/rocprofiler-systems/src/omnitrace/details.cpp +++ b/projects/rocprofiler-systems/src/omnitrace/details.cpp @@ -101,6 +101,14 @@ get_whole_function_names() "daemon", "des_setparity", "div", + "dlopen", + "dlsym", + "dlerror", + "dladdr", + "dlinfo", + "dlvsym", + "dlmopen", + "dl_iterate_phdr", "dysize", "endutxent", "envz_add", diff --git a/projects/rocprofiler-systems/tests/CMakeLists.txt b/projects/rocprofiler-systems/tests/CMakeLists.txt index 854d32f65c..f034325d13 100644 --- a/projects/rocprofiler-systems/tests/CMakeLists.txt +++ b/projects/rocprofiler-systems/tests/CMakeLists.txt @@ -26,7 +26,29 @@ endif() set(_base_environment "OMNITRACE_USE_PERFETTO=ON" "OMNITRACE_USE_TIMEMORY=ON" - "OMNITRACE_USE_SAMPLING=OFF" + "OMNITRACE_USE_SAMPLING=ON" + "OMNITRACE_TIME_OUTPUT=OFF" + "OMP_PROC_BIND=spread" + "OMP_PLACES=threads" + "OMP_NUM_THREADS=2" + "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}:${OMNITRACE_DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}" + ) + +set(_perfetto_environment + "OMNITRACE_USE_PERFETTO=ON" + "OMNITRACE_USE_TIMEMORY=OFF" + "OMNITRACE_USE_SAMPLING=ON" + "OMNITRACE_TIME_OUTPUT=OFF" + "OMP_PROC_BIND=spread" + "OMP_PLACES=threads" + "OMP_NUM_THREADS=2" + "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}:${OMNITRACE_DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}" + ) + +set(_timemory_environment + "OMNITRACE_USE_PERFETTO=OFF" + "OMNITRACE_USE_TIMEMORY=ON" + "OMNITRACE_USE_SAMPLING=ON" "OMNITRACE_TIME_OUTPUT=OFF" "OMP_PROC_BIND=spread" "OMP_PLACES=threads" @@ -48,12 +70,15 @@ set(_fast_environment "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}:${OMNITRACE_DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}" ) +# -------------------------------------------------------------------------------------- # + function(OMNITRACE_ADD_TEST) cmake_parse_arguments( TEST "" # options - "NAME;TARGET;MPI;NUM_PROCS" # single value args - "REWRITE_ARGS;RUNTIME_ARGS;RUN_ARGS;ENVIRONMENT;LABELS" # multiple value args + "NAME;TARGET;MPI;NUM_PROCS;REWRITE_TIMEOUT;RUNTIME_TIMEOUT" # single value args + "REWRITE_ARGS;RUNTIME_ARGS;RUN_ARGS;ENVIRONMENT;LABELS;PROPERTIES" # multiple + # value args ${ARGN}) if("${TEST_MPI}" STREQUAL "") @@ -68,10 +93,20 @@ function(OMNITRACE_ADD_TEST) set(TEST_NUM_PROCS 0) endif() + if(NOT TEST_REWRITE_TIMEOUT) + set(TEST_REWRITE_TIMEOUT 120) + endif() + + if(NOT TEST_RUNTIME_TIMEOUT) + set(TEST_RUNTIME_TIMEOUT 300) + endif() + if(NOT DEFINED TEST_ENVIRONMENT OR "${TEST_ENVIRONMENT}" STREQUAL "") set(TEST_ENVIRONMENT "${_test_environment}") endif() + list(APPEND TEST_ENVIRONMENT "OMNITRACE_CI=ON") + if(TARGET ${TEST_TARGET}) if(DEFINED TEST_MPI AND ${TEST_MPI} @@ -88,14 +123,14 @@ function(OMNITRACE_ADD_TEST) add_test( NAME ${TEST_NAME}-baseline - COMMAND $ ${TEST_RUN_ARGS} + COMMAND ${COMMAND_PREFIX} $ ${TEST_RUN_ARGS} WORKING_DIRECTORY $) add_test( NAME ${TEST_NAME}-binary-rewrite COMMAND $ -o - $/${TEST_TARGET}.inst ${TEST_REWRITE_ARGS} + $/${TEST_NAME}.inst ${TEST_REWRITE_ARGS} -- $ WORKING_DIRECTORY $) @@ -103,20 +138,20 @@ function(OMNITRACE_ADD_TEST) NAME ${TEST_NAME}-binary-rewrite-sampling COMMAND $ -o - $/${TEST_TARGET}.samp -M sampling + $/${TEST_NAME}.samp -M sampling ${TEST_REWRITE_ARGS} -- $ WORKING_DIRECTORY $) add_test( NAME ${TEST_NAME}-binary-rewrite-run - COMMAND ${COMMAND_PREFIX} - $/${TEST_TARGET}.inst ${TEST_RUN_ARGS} + COMMAND ${COMMAND_PREFIX} $/${TEST_NAME}.inst + ${TEST_RUN_ARGS} WORKING_DIRECTORY $) add_test( NAME ${TEST_NAME}-binary-rewrite-run-sampling - COMMAND ${COMMAND_PREFIX} - $/${TEST_TARGET}.samp ${TEST_RUN_ARGS} + COMMAND ${COMMAND_PREFIX} $/${TEST_NAME}.samp + ${TEST_RUN_ARGS} WORKING_DIRECTORY $) add_test( @@ -127,10 +162,8 @@ function(OMNITRACE_ADD_TEST) add_test( NAME ${TEST_NAME}-runtime-instrument-sampling - COMMAND - $ -M sampling --env - OMNITRACE_OUTPUT_PREFIX=sampling- ${TEST_RUNTIME_ARGS} -- - $ ${TEST_RUN_ARGS} + COMMAND $ -M sampling ${TEST_RUNTIME_ARGS} -- + $ ${TEST_RUN_ARGS} WORKING_DIRECTORY $) set_tests_properties(${TEST_NAME}-binary-rewrite-run @@ -143,36 +176,52 @@ function(OMNITRACE_ADD_TEST) _TEST baseline binary-rewrite binary-rewrite-run binary-rewrite-sampling binary-rewrite-run-sampling runtime-instrument runtime-instrument-sampling) - string(REPLACE "-run-" "-" _PREFIX "${TEST_NAME}-${_TEST}-") + string(REPLACE "-run-" "-" _prefix "${TEST_NAME}-${_TEST}-") set(_environ "${TEST_ENVIRONMENT}") + set(_labels "${_TEST}") + set(_timeout ${TEST_REWRITE_TIMEOUT}) list(APPEND _environ "OMNITRACE_OUTPUT_PATH=omnitrace-tests-output" - "OMNITRACE_OUTPUT_PREFIX=${_PREFIX}") - set(_LABELS "${_TEST}") - string(REPLACE "-run" "" _LABELS "${_TEST}") - string(REPLACE "-sampling" ";sampling" _LABELS "${_LABELS}") - set_tests_properties( - ${TEST_NAME}-${_TEST} PROPERTIES ENVIRONMENT "${_environ}" TIMEOUT 600 - LABELS "${_LABELS};${TEST_LABELS}") + "OMNITRACE_OUTPUT_PREFIX=${_prefix}") + string(REPLACE "-run" "" _labels "${_TEST}") + string(REPLACE "-sampling" ";sampling" _labels "${_labels}") + if("${_TEST}" MATCHES "runtime-instrument") + set(_timeout ${TEST_RUNTIME_TIMEOUT}) + endif() + set(_props) + if("${_TEST}" MATCHES "run|baseline") + set(_props ${TEST_PROPERTIES}) + if(NOT "RUN_SERIAL" IN_LIST _props) + list(APPEND _props RUN_SERIAL ON) + endif() + endif() + if(TEST ${TEST_NAME}-${_TEST}) + set_tests_properties( + ${TEST_NAME}-${_TEST} + PROPERTIES ENVIRONMENT "${_environ}" TIMEOUT ${_timeout} LABELS + "${_labels};${TEST_LABELS}" ${_props}) + endif() endforeach() endif() endfunction() +# -------------------------------------------------------------------------------------- # + omnitrace_add_test( NAME transpose TARGET transpose MPI ${TRANSPOSE_USE_MPI} NUM_PROCS ${NUM_PROCS} - REWRITE_ARGS -e -v 1 + REWRITE_ARGS -e -v 2 RUNTIME_ARGS -e -v 1 --label file line return args RUN_ARGS "" - ENVIRONMENT "${_base_environment};OMNITRACE_CRITICAL_TRACE=ON") + ENVIRONMENT "${_base_environment};OMNITRACE_CRITICAL_TRACE=OFF") omnitrace_add_test( NAME transpose-no-save-fpr TARGET transpose MPI ${TRANSPOSE_USE_MPI} NUM_PROCS ${NUM_PROCS} - REWRITE_ARGS -e -v 1 --dyninst-options DelayedParsing TypeChecking + REWRITE_ARGS -e -v 2 --dyninst-options DelayedParsing TypeChecking RUNTIME_ARGS -e -v @@ -191,7 +240,7 @@ omnitrace_add_test( omnitrace_add_test( NAME parallel-overhead TARGET parallel-overhead - REWRITE_ARGS -e -v 1 --min-address-range-loop=64 + REWRITE_ARGS -e -v 2 --min-address-range-loop=64 RUNTIME_ARGS -e -v @@ -202,13 +251,13 @@ omnitrace_add_test( line return args - RUN_ARGS 10 ${NUM_THREADS} + RUN_ARGS 10 ${NUM_THREADS} 1000 ENVIRONMENT "${_base_environment};OMNITRACE_CRITICAL_TRACE=OFF") omnitrace_add_test( NAME parallel-overhead-no-save-fpr TARGET parallel-overhead - REWRITE_ARGS -e -v 1 --min-address-range-loop=32 --dyninst-options DelayedParsing + REWRITE_ARGS -e -v 2 --min-address-range-loop=32 --dyninst-options DelayedParsing TypeChecking RUNTIME_ARGS -e @@ -223,7 +272,7 @@ omnitrace_add_test( --dyninst-options DelayedParsing TypeChecking - RUN_ARGS 10 ${NUM_THREADS} + RUN_ARGS 10 ${NUM_THREADS} 1000 ENVIRONMENT "${_fast_environment}") omnitrace_add_test( @@ -231,7 +280,7 @@ omnitrace_add_test( TARGET lulesh MPI ${LULESH_USE_MPI} NUM_PROCS 8 - REWRITE_ARGS -e -v 1 + REWRITE_ARGS -e -v 2 RUNTIME_ARGS -e -v @@ -242,6 +291,69 @@ omnitrace_add_test( return args -ME - [==['lib(gomp|m-)']==] + [==[lib(gomp|m-)]==] RUN_ARGS -i 10 -s 20 -p - ENVIRONMENT "${_base_environment};OMNITRACE_CRITICAL_TRACE=OFF") + ENVIRONMENT + "${_base_environment};OMNITRACE_CRITICAL_TRACE=OFF;OMNITRACE_USE_KOKKOSP=OFF") + +omnitrace_add_test( + NAME lulesh-kokkosp + TARGET lulesh + MPI ${LULESH_USE_MPI} + NUM_PROCS 8 + REWRITE_ARGS -e -v 2 + RUNTIME_ARGS + -e + -v + 1 + --label + file + line + return + args + -ME + [==[lib(gomp|m-)]==] + RUN_ARGS -i 10 -s 20 -p + ENVIRONMENT + "${_base_environment};OMNITRACE_CRITICAL_TRACE=OFF;OMNITRACE_USE_KOKKOSP=ON") + +omnitrace_add_test( + NAME lulesh-perfetto + TARGET lulesh + MPI ${LULESH_USE_MPI} + NUM_PROCS 8 + REWRITE_ARGS -e -v 2 + RUNTIME_ARGS + -e + -v + 1 + -l + --dynamic-callsites + --traps + --allow-overlapping + -ME + [==[libgomp]==] + RUN_ARGS -i 10 -s 20 -p + ENVIRONMENT + "${_perfetto_environment};OMNITRACE_CRITICAL_TRACE=OFF;OMNITRACE_USE_KOKKOSP=OFF") + +omnitrace_add_test( + NAME lulesh-timemory + TARGET lulesh + MPI ${LULESH_USE_MPI} + NUM_PROCS 8 + REWRITE_ARGS -e -v 2 -l --dynamic-callsites --traps --allow-overlapping + RUNTIME_ARGS + -e + -v + 1 + -l + --dynamic-callsites + -ME + [==[libgomp]==] + -d + wall_clock + peak_rss + RUN_ARGS -i 10 -s 20 -p + ENVIRONMENT + "${_timemory_environment};OMNITRACE_CRITICAL_TRACE=OFF;OMNITRACE_USE_KOKKOSP=OFF")