From 1688a027d8a28e68ec52f0ec68e64ea4264b76f9 Mon Sep 17 00:00:00 2001 From: "Jonathan R. Madsen" Date: Tue, 7 Mar 2023 06:04:19 -0600 Subject: [PATCH] Add RedHat CI and release packaging (#251) - additional miscellaneous tweaks to workflows and docker scripts, e.g. install perfetto python bindings - improves the stability of MPI finalization - reduces some debug messages within timemory when `OMNITRACE_DEBUG=ON` - fixes issue found in RHEL where libunwind is using mutex and omnitrace was not treating this as an internal mutex call - this may have been affecting the causal profiling slightly (tests seem a bit more stable now) - fix data race in timemory * Add RedHat CI and release packaging - additional miscellaneous tweaks to workflows and docker scripts, e.g. install perfetto python bindings * Fix URL for ROCm packages in redhat workflow * Fix dnf --enable-repo for ROCm perl packages * Dockerfile.rhel and redhat.yml updates - Fix dnf repo for ROCm PERL packages - Disable python in CI (interpreter segfaults) - Exclude parallel-overhead-locks tests due to inclusion of internal locks - This needs to be remedied in the future * Exclude _dl_relocate_static_pie from instrumentation * Testing updates - OMNITRACE_SAMPLING_KEEP_INTERNAL=OFF for parallel-overhead-locks * Fix redhat workflow * redhat.yml update - remove if condition on config/build/test step * Update timemory submodule - tweaks to verbosity messages * Set thread state before unw_step - on Redhat, unw_step calls mutex * Update timemory submodule - verbosity changes - gotcha uses spin_lock/spin_mutex * Remove using gsplit-dwarf unless OMNITRACE_BUILD_NUMBER > 2 * Re-enable parallel-overhead-locks tests in redhat workflow * Always disable timemory manager metadata auto output * testing updates - tweak parallel-overhead-locks-timemory to higher instruction count min - OMNITRACE_SAMPLING_KEEP_INTERNAL=OFF for parallel-overhead-locks-perfetto * Update timemory submodule - quiet realpath queries * omnitrace exe updates - detect text files - improved bin/lib locating * cmake format * test-install.sh and redhat workflow updates - handle testing when ls is script - re-enable python testing on redhat workflow - invoke test-install.sh in redhat workflow * Misc guards for finalization * omnitrace-exe, testing updates - test-install.sh: LS_EXEC -> LS_NAME - handle /usr/bin/ls being script in source/bin/tests - improve locating the binary * Fix mpi_gotcha compile error * omnitrace-exe updates - improve file locating * formatting * Misc fixes - remove -static-libstdc++ for RHEL packaging (rocky-linux doesn't distribute static lib) * omnitrace-exe paths * Replace realpath with absolute - using absolute path to symlink fixes issues with locating libdyninstAPI_RT at runtime * omnitrace exe updates - judicious use of realpath * Update timemory submodule - fix update main hash ids/aliases data race in merge * bin tests update - change working directory of omnitrace-exe-simulate-lib-basename * omnitrace exe updates - Update resolved exe/lib messaging * bin tests update - change working directory of omnitrace-exe-simulate-lib-basename --- .github/workflows/containers.yml | 6 + .github/workflows/cpack.yml | 30 ++ .github/workflows/opensuse.yml | 2 +- .github/workflows/redhat.yml | 139 +++++ .github/workflows/ubuntu-bionic.yml | 1 - .github/workflows/ubuntu-focal.yml | 14 +- cmake/BuildSettings.cmake | 2 +- cmake/Perfetto.cmake | 6 +- docker/Dockerfile.centos | 85 --- docker/Dockerfile.opensuse.ci | 3 +- docker/Dockerfile.rhel | 52 ++ docker/Dockerfile.rhel.ci | 51 ++ docker/Dockerfile.ubuntu.ci | 3 +- docker/build-docker-ci.sh | 14 +- docker/build-docker-release.sh | 10 +- docker/build-docker.sh | 57 +- ...ntrypoint-centos.sh => entrypoint-rhel.sh} | 3 - external/timemory | 2 +- scripts/test-install.sh | 26 +- source/bin/omnitrace/details.cpp | 204 ++++++- source/bin/omnitrace/log.hpp | 2 +- source/bin/omnitrace/omnitrace.cpp | 500 ++++++++++-------- source/bin/omnitrace/omnitrace.hpp | 61 ++- source/bin/tests/CMakeLists.txt | 44 +- source/lib/omnitrace/library.cpp | 4 +- .../library/causal/components/backtrace.cpp | 2 + .../library/components/backtrace.cpp | 4 + .../library/components/mpi_gotcha.cpp | 10 +- .../components/pthread_create_gotcha.cpp | 7 + tests/CMakeLists.txt | 9 +- 30 files changed, 958 insertions(+), 395 deletions(-) create mode 100644 .github/workflows/redhat.yml delete mode 100644 docker/Dockerfile.centos create mode 100644 docker/Dockerfile.rhel create mode 100644 docker/Dockerfile.rhel.ci rename docker/{entrypoint-centos.sh => entrypoint-rhel.sh} (66%) diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml index 0135914772..66ea47e069 100644 --- a/.github/workflows/containers.yml +++ b/.github/workflows/containers.yml @@ -36,6 +36,12 @@ jobs: version: "15.3" - distro: "opensuse" version: "15.4" + - distro: "rhel" + version: "8.7" + - distro: "rhel" + version: "9.0" + - distro: "rhel" + version: "9.1" steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/cpack.yml b/.github/workflows/cpack.yml index 93472a3ecf..94423260da 100644 --- a/.github/workflows/cpack.yml +++ b/.github/workflows/cpack.yml @@ -115,6 +115,36 @@ jobs: - os-distro: "opensuse" os-version: "15.4" rocm-version: "5.4" + # RHEL 8.7 + - os-distro: "rhel" + os-version: "8.7" + rocm-version: "0.0" + - os-distro: "rhel" + os-version: "8.7" + rocm-version: "5.3" + - os-distro: "rhel" + os-version: "8.7" + rocm-version: "5.4" + # RHEL 9.0 + - os-distro: "rhel" + os-version: "9.0" + rocm-version: "0.0" + - os-distro: "rhel" + os-version: "9.0" + rocm-version: "5.3" + - os-distro: "rhel" + os-version: "9.0" + rocm-version: "5.4" + # RHEL 9.1 + - os-distro: "rhel" + os-version: "9.1" + rocm-version: "0.0" + - os-distro: "rhel" + os-version: "9.1" + rocm-version: "5.3" + - os-distro: "rhel" + os-version: "9.1" + rocm-version: "5.4" steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/opensuse.yml b/.github/workflows/opensuse.yml index 7a179ce564..f4806b6a69 100644 --- a/.github/workflows/opensuse.yml +++ b/.github/workflows/opensuse.yml @@ -30,7 +30,7 @@ jobs: fail-fast: false matrix: compiler: ['g++'] - os-release: [ '15.2', '15.3' ] + os-release: [ '15.2', '15.3', '15.4' ] build-type: ['Release'] steps: diff --git a/.github/workflows/redhat.yml b/.github/workflows/redhat.yml new file mode 100644 index 0000000000..d7ea79e7cd --- /dev/null +++ b/.github/workflows/redhat.yml @@ -0,0 +1,139 @@ +name: RedHat Linux (GCC, Python, ROCm) + +on: + push: + branches: [ main, develop ] + paths-ignore: + - '*.md' + - 'source/docs/**' + pull_request: + branches: [ main, develop ] + paths-ignore: + - '*.md' + - 'source/docs/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + OMNITRACE_CI: ON + OMNITRACE_TMPDIR: "%env{PWD}%/testing-tmp" + +jobs: + rhel: + runs-on: ubuntu-20.04 + container: + image: jrmadsen/omnitrace:ci-base-rhel-${{ matrix.os-release }} + strategy: + fail-fast: false + matrix: + compiler: ['g++'] + os-release: [ '8.7', '9.0', '9.1' ] + rocm-version: [ '0.0', '5.3', '5.4' ] + build-type: ['Release'] + + steps: + - uses: actions/checkout@v3 + + - name: Configure Env + shell: bash + run: + echo "CC=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')" >> $GITHUB_ENV && + echo "CXX=${{ matrix.compiler }}" >> $GITHUB_ENV && + env + + - name: Install Packages + if: ${{ matrix.rocm-version > 0 }} + timeout-minutes: 30 + shell: bash + run: | + OS_VERSION_MAJOR=$(cat /etc/os-release | grep 'VERSION_ID' | sed 's/=/ /1' | awk '{print $NF}' | sed 's/"//g' | sed 's/\./ /g' | awk '{print $1}') + RPM_TAG=".el${OS_VERSION_MAJOR}" + ROCM_VERSION=${{ matrix.rocm-version }} + ROCM_MAJOR=$(echo ${ROCM_VERSION} | sed 's/\./ /g' | awk '{print $1}') + ROCM_MINOR=$(echo ${ROCM_VERSION} | sed 's/\./ /g' | awk '{print $2}') + ROCM_VERSN=$(( (${ROCM_MAJOR}*10000)+(${ROCM_MINOR}*100) )) + if [ "${OS_VERSION_MAJOR}" -eq 8 ]; then PERL_REPO=powertools; else PERL_REPO=crb; fi && \ + dnf -y --enablerepo=${PERL_REPO} install perl-File-BaseDir + yum install -y https://repo.radeon.com/amdgpu-install/${{ matrix.rocm-version }}/rhel/${{ matrix.os-release }}/amdgpu-install-${ROCM_MAJOR}.${ROCM_MINOR}.${ROCM_VERSN}-1${RPM_TAG}.noarch.rpm + amdgpu-install --usecase=rocm,hip,hiplibsdk --no-dkms --skip-broken -y + yum install -y rocm-hip-sdk rocm-smi-lib roctracer-dev rocprofiler-dev + + - name: Configure, Build, and Test + timeout-minutes: 115 + shell: bash + run: + git config --global --add safe.directory ${PWD} && + cmake --version && + TAG="${{ github.repository_owner }}-${{ github.ref_name }}-rhel-${{ matrix.os-release }}-${{ matrix.compiler }}-python-mpip" && + USE_HIP=OFF && + if [ ${{ matrix.rocm-version }} != "0.0" ]; then USE_HIP=ON; TAG="${TAG}-rocm-${{ matrix.rocm-version }}"; fi && + python3 ./scripts/run-ci.py -B build + --name ${TAG} + --build-jobs 2 + --site GitHub + -- + -DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g') + -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} + -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} + -DCMAKE_INSTALL_PREFIX=/opt/omnitrace + -DOMNITRACE_BUILD_TESTING=ON + -DOMNITRACE_USE_MPI=OFF + -DOMNITRACE_USE_HIP=${USE_HIP} + -DOMNITRACE_USE_OMPT=OFF + -DOMNITRACE_USE_PYTHON=ON + -DOMNITRACE_USE_MPI_HEADERS=ON + -DOMNITRACE_CI_MPI_RUN_AS_ROOT=ON + -DOMNITRACE_MAX_THREADS=64 + -DOMNITRACE_PYTHON_PREFIX=/opt/conda/envs + -DOMNITRACE_PYTHON_ENVS="py3.6;py3.7;py3.8;py3.9;py3.10" + -DOMNITRACE_DISABLE_EXAMPLES="transpose;rccl" + -DOMNITRACE_BUILD_NUMBER=${{ github.run_attempt }} + -- + -LE "transpose|rccl" + + - name: Install + timeout-minutes: 10 + run: + cmake --build build --target install --parallel 2 + + - name: Test Install + timeout-minutes: 10 + shell: bash + run: | + set -v + source /opt/omnitrace/share/omnitrace/setup-env.sh + ./scripts/test-install.sh --test-omnitrace{,-avail,-sample,-rewrite,-runtime,-critical-trace,-python}=1 + + - name: Test User API + timeout-minutes: 10 + run: | + set -v + ./scripts/test-find-package.sh --install-dir /opt/omnitrace + + - name: CTest Artifacts + if: success() || failure() + uses: actions/upload-artifact@v3 + with: + name: ctest-${{ github.job }}-log + path: | + build/*.log + + - name: Data Artifacts + if: success() || failure() + uses: actions/upload-artifact@v3 + with: + name: data-${{ github.job }}-files + path: | + build/omnitrace-tests-config/*.cfg + build/omnitrace-tests-output/**/*.txt + build/omnitrace-tests-output/**/*-instr*.json + + - name: Kill Perfetto + if: success() || failure() + continue-on-error: True + run: | + set +e + RUNNING_PROCS=$(pgrep trace_processor_shell) + if [ -n "${RUNNING_PROCS}" ]; then kill -s 9 ${RUNNING_PROCS}; fi diff --git a/.github/workflows/ubuntu-bionic.yml b/.github/workflows/ubuntu-bionic.yml index 75f8329c8b..2226f782f7 100644 --- a/.github/workflows/ubuntu-bionic.yml +++ b/.github/workflows/ubuntu-bionic.yml @@ -19,7 +19,6 @@ concurrency: env: BUILD_TYPE: Release - ELFUTILS_DOWNLOAD_VERSION: 0.183 OMNITRACE_CI: ON GIT_DISCOVERY_ACROSS_FILESYSTEM: 1 OMNITRACE_TMPDIR: "%env{PWD}%/testing-tmp" diff --git a/.github/workflows/ubuntu-focal.yml b/.github/workflows/ubuntu-focal.yml index 3c1336191c..116a67720b 100644 --- a/.github/workflows/ubuntu-focal.yml +++ b/.github/workflows/ubuntu-focal.yml @@ -79,7 +79,7 @@ jobs: add-apt-repository -y ppa:ubuntu-toolchain-r/test && apt-get update && apt-get upgrade -y && - apt-get install -y build-essential m4 autoconf libtool python3-pip libiberty-dev clang libomp-dev libmpich-dev mpich environment-modules ${{ matrix.compiler }} && + apt-get install -y build-essential m4 autoconf libtool python3-pip libiberty-dev clang libmpich-dev mpich environment-modules ${{ matrix.compiler }} && python3 -m pip install --upgrade pip && python3 -m pip install numpy && python3 -m pip install perfetto && @@ -222,12 +222,18 @@ jobs: fail-fast: false matrix: compiler: ['g++'] - rocm-version: ['4.3', '4.5', '5.0'] + rocm-version: ['4.5', '5.0', '5.1', '5.2', '5.3'] mpi-headers: ['OFF'] - build-jobs: ['4'] + build-jobs: ['3'] ctest-exclude: ['-LE "mpi-example|transpose"'] - perfetto-tools: ['ON'] + perfetto-tools: ['OFF'] include: + - compiler: 'g++' + rocm-version: '4.3' + mpi-headers: 'ON' + build-jobs: '2' + ctest-exclude: '-LE transpose' + perfetto-tools: 'ON' - compiler: 'g++' rocm-version: 'debian' mpi-headers: 'ON' diff --git a/cmake/BuildSettings.cmake b/cmake/BuildSettings.cmake index dad4c7b936..ca5dc8c826 100644 --- a/cmake/BuildSettings.cmake +++ b/cmake/BuildSettings.cmake @@ -233,7 +233,7 @@ if(OMNITRACE_BUILD_DEVELOPER) omnitrace-compile-options "-Werror" "-Wdouble-promotion" "-Wshadow" "-Wextra" "-Wpedantic" "-Wstack-usage=524288" # 512 KB "/showIncludes") - if(OMNITRACE_BUILD_NUMBER LESS 2) + if(OMNITRACE_BUILD_NUMBER GREATER 2) add_target_flag_if_avail(omnitrace-compile-options "-gsplit-dwarf") endif() endif() diff --git a/cmake/Perfetto.cmake b/cmake/Perfetto.cmake index e3ef045bc2..7e8e490f43 100644 --- a/cmake/Perfetto.cmake +++ b/cmake/Perfetto.cmake @@ -50,7 +50,7 @@ set(OMNITRACE_PERFETTO_BINARY_DIR set(OMNITRACE_PERFETTO_INSTALL_DIR ${PROJECT_BINARY_DIR}/external/perfetto/source/out/linux/stripped) set(OMNITRACE_PERFETTO_LINK_FLAGS - "-static-libgcc -static-libstdc++" + "-static-libgcc" CACHE STRING "Link flags for perfetto") set(OMNITRACE_PERFETTO_BUILD_THREADS ${_NUM_THREADS} @@ -67,10 +67,10 @@ if(CMAKE_CXX_COMPILER_IS_CLANG) else() set(PERFETTO_IS_CLANG false) set(OMNITRACE_PERFETTO_C_FLAGS - "-static-libgcc -static-libstdc++ -Wno-maybe-uninitialized" + "-static-libgcc -Wno-maybe-uninitialized" CACHE STRING "Perfetto C flags") set(OMNITRACE_PERFETTO_CXX_FLAGS - "-static-libgcc -static-libstdc++ -Wno-maybe-uninitialized" + "-static-libgcc -Wno-maybe-uninitialized" CACHE STRING "Perfetto C++ flags") endif() diff --git a/docker/Dockerfile.centos b/docker/Dockerfile.centos deleted file mode 100644 index c1bcc32203..0000000000 --- a/docker/Dockerfile.centos +++ /dev/null @@ -1,85 +0,0 @@ -ARG DISTRO=centos -ARG VERSION=7 -FROM ${DISTRO}:${VERSION} - -ENV HOME /root -ENV SHELL /bin/bash -ENV BASH_ENV /etc/bash.bashrc -ENV DEBIAN_FRONTEND noninteractive - -WORKDIR /tmp -SHELL [ "/bin/bash", "-c" ] - -ENV PATH /usr/local/bin:${PATH} - -RUN if [ -n "$(cat /etc/os-release | grep 'VERSION="8"')" ]; then \ - dnf -y --disablerepo '*' --enablerepo=extras swap centos-linux-repos centos-stream-repos && \ - dnf -y distro-sync; \ - fi - -RUN yum update -y && \ - yum groupinstall -y "Development Tools" - -ARG TOOLSET_VERSION=9 - -RUN yum install -y epel-release && \ - if [ -n "$(cat /etc/os-release | grep 'VERSION="8"')" ]; then \ - yum install -y centos-release-stream && \ - yum install -y gcc-toolset-${TOOLSET_VERSION} openmpi-devel; \ - else \ - yum install -y centos-release-scl && \ - yum install -y devtoolset-${TOOLSET_VERSION} openmpi3-devel dpkg-dev; \ - fi; \ - yum install -y python3-pip zlib-devel numactl-devel papi-devel dpkg-devel wget curl && \ - python3 -m pip install 'cmake==3.21.4' - -ARG ROCM_VERSION=0.0 -ARG AMDGPU_RPM=21.40.2/rhel/7.9/amdgpu-install-21.40.2.40502-1.el7.noarch.rpm -# ARG AMDGPU_RPM=latest/rhel/7.9/amdgpu-install-21.50.50000-1.el7.noarch.rpm - -RUN if [ "${ROCM_VERSION}" != "0.0" ]; then \ - yum install -y https://repo.radeon.com/amdgpu-install/${AMDGPU_RPM} && \ - amdgpu-install --usecase=rocm,hip,hiplibsdk --no-dkms --skip-broken -y && \ - yum install -y rocm-hip-sdk rocm-smi-lib roctracer-dev rocprofiler-dev rccl-dev && \ - yum update -y && \ - yum clean all; \ - fi - -ARG PYTHON_VERSIONS="6 7 8 9 10" - -RUN wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh && \ - bash miniconda.sh -b -p /opt/conda && \ - export PATH="/opt/conda/bin:${PATH}" && \ - conda config --set always_yes yes --set changeps1 no && \ - conda update -c defaults -n base conda && \ - for i in ${PYTHON_VERSIONS}; do conda create -n py3.${i} -c defaults python=3.${i} pip; done && \ - conda clean -a -y && \ - conda init - -RUN if [ "${ROCM_VERSION}" != "0.0" ]; then ln -sf /opt/rocm-${ROCM_VERSION}* /opt/rocm; fi - -WORKDIR /home -SHELL [ "/bin/bash", "--login", "-c" ] -COPY ./entrypoint-centos.sh /docker-entrypoint.sh -ENTRYPOINT [ "/docker-entrypoint.sh" ] - -#1 yum update -#2 yum groupinstall "Development Tools" -#3 yum install devtoolset-9-toolchain -#4 yum install devtoolset-9 -#5 yum install devtoolset-7-toolchain -#6 yum search devtoolset -#7 yum search -a devtoolset -#8 yum search --help -#9 yum repolist -#10 yum list available -#11 yum list available devtoolset* -#12 yum list available devtoolset\* -#13 subscription-manager list --available -#14 yum install subscription-manager -#15 subscription-manager list --available -#16 yum install centos-release-scl -#17 yum-config-manager --enable rhel-server-rhscl-7-rpms -#18 yum install devtoolset-7 -#19 yum install devtoolset-9 -#20 scl enable devtoolset-9 bash diff --git a/docker/Dockerfile.opensuse.ci b/docker/Dockerfile.opensuse.ci index 6c89739a0c..68a4f98267 100644 --- a/docker/Dockerfile.opensuse.ci +++ b/docker/Dockerfile.opensuse.ci @@ -13,7 +13,7 @@ SHELL [ "/bin/bash", "-c" ] ENV PATH /usr/local/bin:${PATH} ARG EXTRA_PACKAGES="" -ARG ELFUTILS_DOWNLOAD_VERSION="0.183" +ARG ELFUTILS_DOWNLOAD_VERSION="0.186" ARG NJOBS="12" RUN zypper update -y && \ @@ -42,6 +42,7 @@ RUN wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh - conda update -c defaults -n base conda && \ for i in ${PYTHON_VERSIONS}; do conda create -n py3.${i} -c defaults python=3.${i} pip numpy; done && \ conda clean -a -y && \ + for i in ${PYTHON_VERSIONS}; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done && \ cd /tmp && \ shopt -s dotglob extglob && \ rm -rf * diff --git a/docker/Dockerfile.rhel b/docker/Dockerfile.rhel new file mode 100644 index 0000000000..8f4bf66373 --- /dev/null +++ b/docker/Dockerfile.rhel @@ -0,0 +1,52 @@ +ARG DISTRO=rockylinux +ARG VERSION=8 +FROM ${DISTRO}:${VERSION} + +ENV HOME /root +ENV SHELL /bin/bash +ENV BASH_ENV /etc/bash.bashrc +ENV DEBIAN_FRONTEND noninteractive + +WORKDIR /tmp +SHELL [ "/bin/bash", "-c" ] + +ENV PATH /usr/local/bin:${PATH} + +RUN yum update -y && \ + yum groupinstall -y "Development Tools" && \ + yum install -y epel-release && \ + yum install -y --allowerasing curl dpkg-devel numactl-devel openmpi-devel papi-devel python3-pip wget zlib-devel && \ + yum clean all && \ + python3 -m pip install 'cmake==3.21.4' + +ARG ROCM_VERSION=0.0 +ARG AMDGPU_RPM=5.4/rhel/8.7/amdgpu-install-5.4.50400-1.el8.noarch.rpm + +RUN if [ "${ROCM_VERSION}" != "0.0" ]; then \ + OS_VERSION_MAJOR=$(cat /etc/os-release | grep 'VERSION_ID' | sed 's/=/ /1' | awk '{print $NF}' | sed 's/"//g' | sed 's/\./ /g' | awk '{print $1}') && \ + yum update -y && \ + if [ "${OS_VERSION_MAJOR}" -eq 8 ]; then PERL_REPO=powertools; else PERL_REPO=crb; fi && \ + dnf -y --enablerepo=${PERL_REPO} install perl-File-BaseDir && \ + yum install -y https://repo.radeon.com/amdgpu-install/${AMDGPU_RPM} && \ + amdgpu-install --usecase=rocm,hip,hiplibsdk --no-dkms --skip-broken -y && \ + yum install -y rocm-hip-sdk rocm-smi-lib roctracer-dev rocprofiler-dev && \ + yum update -y && \ + yum clean all; \ + fi + +ARG PYTHON_VERSIONS="6 7 8 9 10" + +RUN wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh && \ + bash miniconda.sh -b -p /opt/conda && \ + export PATH="/opt/conda/bin:${PATH}" && \ + conda config --set always_yes yes --set changeps1 no && \ + conda update -c defaults -n base conda && \ + for i in ${PYTHON_VERSIONS}; do conda create -n py3.${i} -c defaults python=3.${i} pip; done && \ + conda clean -a -y && \ + conda init + +RUN if [ "${ROCM_VERSION}" != "0.0" ]; then ln -sf /opt/rocm-${ROCM_VERSION}* /opt/rocm; fi + +WORKDIR /home +ENV LC_ALL C.UTF-8 +SHELL [ "/bin/bash", "--login", "-c" ] diff --git a/docker/Dockerfile.rhel.ci b/docker/Dockerfile.rhel.ci new file mode 100644 index 0000000000..fcf348a97e --- /dev/null +++ b/docker/Dockerfile.rhel.ci @@ -0,0 +1,51 @@ +ARG DISTRO=rockylinux +ARG VERSION=8 +FROM ${DISTRO}:${VERSION} + +ENV HOME /root +ENV SHELL /bin/bash +ENV BASH_ENV /etc/bash.bashrc +ENV DEBIAN_FRONTEND noninteractive + +WORKDIR /tmp +SHELL [ "/bin/bash", "-c" ] + +ENV PATH /usr/local/bin:${PATH} + +ARG EXTRA_PACKAGES="" +ARG ELFUTILS_DOWNLOAD_VERSION="0.186" +ARG NJOBS="12" + +RUN yum update -y && \ + yum groupinstall -y "Development Tools" && \ + yum install -y epel-release && \ + yum install -y --allowerasing curl dpkg-devel numactl-devel openmpi-devel papi-devel python3-pip wget zlib-devel && \ + yum clean all && \ + python3 -m pip install 'cmake==3.21.4' + +COPY ./dyninst-source /tmp/dyninst + +RUN cd /tmp/dyninst && \ + cmake -B build -DCMAKE_BUILD_TYPE=RelWithDebInfo -DBUILD_BOOST=ON -DBUILD_TBB=ON -DBUILD_ELFUTILS=ON -DBUILD_LIBIBERTY=ON && \ + cmake --build build --target all --parallel ${NJOBS} && \ + cmake --build build --target install --parallel ${NJOBS} && \ + cd /tmp && \ + shopt -s dotglob extglob && \ + rm -rf * + +ARG PYTHON_VERSIONS="6 7 8 9 10" + +RUN wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh && \ + bash miniconda.sh -b -p /opt/conda && \ + export PATH="/opt/conda/bin:${PATH}" && \ + conda config --set always_yes yes --set changeps1 no && \ + conda update -c defaults -n base conda && \ + for i in ${PYTHON_VERSIONS}; do conda create -n py3.${i} -c defaults python=3.${i} pip numpy; done && \ + conda clean -a -y && \ + for i in ${PYTHON_VERSIONS}; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done && \ + cd /tmp && \ + shopt -s dotglob extglob && \ + rm -rf * + +WORKDIR /home +SHELL [ "/bin/bash", "--login", "-c" ] diff --git a/docker/Dockerfile.ubuntu.ci b/docker/Dockerfile.ubuntu.ci index 47e5909a63..0446bec1f5 100644 --- a/docker/Dockerfile.ubuntu.ci +++ b/docker/Dockerfile.ubuntu.ci @@ -27,7 +27,7 @@ COPY ./dyninst-source /tmp/dyninst RUN apt-get update && \ apt-get dist-upgrade -y && \ - apt-get install -y build-essential cmake wget gnupg2 m4 bash-completion git-core autoconf libtool autotools-dev python3-pip lsb-release zlib1g-dev libpapi-dev libpfm4-dev zip unzip locales bzip2 gzip curl && \ + apt-get install -y autoconf autotools-dev bash-completion build-essential bzip2 clang cmake curl environment-modules git-core gnupg2 gzip libiberty-dev libmpich-dev libpapi-dev libpfm4-dev libtool locales lsb-release m4 mpich python3-pip unzip wget zip zlib1g-dev && \ python3 -m pip install 'cmake==3.18.4' && \ apt-get autoclean && \ locale -a && \ @@ -42,6 +42,7 @@ RUN apt-get update && \ conda update -c defaults -n base conda && \ for i in ${PYTHON_VERSIONS}; do conda create -n py3.${i} -c defaults python=3.${i} pip numpy; done && \ conda clean -a -y && \ + for i in ${PYTHON_VERSIONS}; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done && \ cd /tmp && \ shopt -s dotglob extglob && \ rm -rf * diff --git a/docker/build-docker-ci.sh b/docker/build-docker-ci.sh index fc7b98dc67..fd1f0e1908 100755 --- a/docker/build-docker-ci.sh +++ b/docker/build-docker-ci.sh @@ -35,8 +35,8 @@ usage() echo "" print_default_option() { printf " --%-20s %-24s %s (default: %s)\n" "${1}" "${2}" "${3}" "$(tolower ${4})"; } - print_default_option distro "[ubuntu|opensuse]" "OS distribution" "${DISTRO}" - print_default_option versions "[VERSION] [VERSION...]" "Ubuntu or OpenSUSE release" "${VERSIONS}" + print_default_option distro "[ubuntu|opensuse|rhel]" "OS distribution" "${DISTRO}" + print_default_option versions "[VERSION] [VERSION...]" "Ubuntu, OpenSUSE, or RHEL release" "${VERSIONS}" print_default_option "jobs -j" "[N]" "parallel build jobs" "${NJOBS}" print_default_option elfutils-version "[0.183..0.186]" "ElfUtils version" "${ELFUTILS_VERSION}" print_default_option boost-version "[1.67.0..1.79.0]" "Boost version" "${BOOST_VERSION}" @@ -126,9 +126,13 @@ verbose-run rm -rf ./dyninst-source/{build,install}* set -e -DISTRO_IMAGE=${DISTRO} - -if [ "${DISTRO}" = "opensuse" ]; then DISTRO_IMAGE="opensuse/leap"; fi +if [ "${DISTRO}" = "opensuse" ]; then + DISTRO_IMAGE="opensuse/leap" +elif [ "${DISTRO}" = "rhel" ]; then + DISTRO_IMAGE="rockylinux" +else + DISTRO_IMAGE=${DISTRO} +fi for VERSION in ${VERSIONS} do diff --git a/docker/build-docker-release.sh b/docker/build-docker-release.sh index 147e84b0c1..4b779645d4 100755 --- a/docker/build-docker-release.sh +++ b/docker/build-docker-release.sh @@ -13,12 +13,12 @@ set -e tolower() { - echo "$@" | awk -F '\|~\|' '{print tolower($1)}'; + echo "$@" | awk -F '\\|~\\|' '{print tolower($1)}'; } toupper() { - echo "$@" | awk -F '\|~\|' '{print toupper($1)}'; + echo "$@" | awk -F '\\|~\\|' '{print toupper($1)}'; } usage() @@ -29,7 +29,7 @@ usage() echo "" print_default_option() { printf " --%-20s %-24s %s (default: %s)\n" "${1}" "${2}" "${3}" "$(tolower ${4})"; } - print_default_option distro "[ubuntu|opensuse]" "OS distribution" "${DISTRO}" + print_default_option distro "[ubuntu|opensuse|rhel]" "OS distribution" "${DISTRO}" print_default_option versions "[VERSION] [VERSION...]" "Ubuntu or OpenSUSE release" "${VERSIONS}" print_default_option rocm-versions "[VERSION] [VERSION...]" "ROCm versions" "${ROCM_VERSIONS}" print_default_option python-versions "[VERSION] [VERSION...]" "Python 3 minor releases" "${PYTHON_VERSIONS}" @@ -157,6 +157,10 @@ done CODE_VERSION=$(cat VERSION) +if [ "${DISTRO}" = "rhel" ]; then + SCRIPT_ARGS="${SCRIPT_ARGS} --static-libstdcxx off" +fi + for VERSION in ${VERSIONS} do TAG=${DISTRO}-${VERSION} diff --git a/docker/build-docker.sh b/docker/build-docker.sh index 6bde52129a..33ace904a8 100755 --- a/docker/build-docker.sh +++ b/docker/build-docker.sh @@ -29,8 +29,8 @@ usage() echo "" print_default_option() { printf " --%-20s %-24s %s (default: %s)\n" "${1}" "${2}" "${3}" "$(tolower ${4})"; } - print_default_option distro "[ubuntu|opensuse]" "OS distribution" "${DISTRO}" - print_default_option versions "[VERSION] [VERSION...]" "Ubuntu or OpenSUSE release" "${VERSIONS}" + print_default_option distro "[ubuntu|opensuse|rhel]" "OS distribution" "${DISTRO}" + print_default_option versions "[VERSION] [VERSION...]" "Ubuntu, OpenSUSE, or RHEL release" "${VERSIONS}" print_default_option rocm-versions "[VERSION] [VERSION...]" "ROCm versions" "${ROCM_VERSIONS}" print_default_option python-versions "[VERSION] [VERSION...]" "Python 3 minor releases" "${PYTHON_VERSIONS}" print_default_option "user -u" "[USERNAME]" "DockerHub username" "${USER}" @@ -185,41 +185,26 @@ do ;; esac verbose-build docker build . -f ${DOCKER_FILE} --tag ${CONTAINER} --build-arg DISTRO=${DISTRO} --build-arg VERSION=${VERSION} --build-arg ROCM_VERSION=${ROCM_VERSION} --build-arg ROCM_REPO_VERSION=${ROCM_REPO_VERSION} --build-arg ROCM_REPO_DIST=${ROCM_REPO_DIST} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\" - elif [ "${DISTRO}" = "centos" ]; then - case "${VERSION}" in - 7) - RPM_PATH=7.9 - RPM_TAG=".el7" - TOOLSET_VERSION=9 - ;; - 8) - RPM_PATH=8.4 - RPM_TAG=".el8" - TOOLSET_VERSION=11 - ;; - 9) - RPM_PATH=9.0 - RPM_TAG=".el9" - TOOLSET_VERSION=11 - ;; - *) - send-error "Invalid centos version ${VERSION}. Supported: 7, 8, 9" - esac + elif [ "${DISTRO}" = "rhel" ]; then + if [ -z "${VERSION_MINOR}" ]; then + send-error "Please provide a major and minor version of the OS. Supported: >= 8.7, <= 9.1" + fi + + # Components used to create the sub-URL below + # set in amdgpu-install//rhel/ + RPM_PATH=${VERSION_MAJOR}.${VERSION_MINOR} + RPM_TAG=".el${VERSION_MAJOR}" + + # set the sub-URL in https://repo.radeon.com/amdgpu-install/ case "${ROCM_VERSION}" in + 5.4 | 5.4.*) + ROCM_RPM=${ROCM_VERSION}/rhel/${RPM_PATH}/amdgpu-install-${ROCM_MAJOR}.${ROCM_MINOR}.${ROCM_VERSN}-1${RPM_TAG}.noarch.rpm + ;; 5.3 | 5.3.*) ROCM_RPM=${ROCM_VERSION}/rhel/${RPM_PATH}/amdgpu-install-${ROCM_MAJOR}.${ROCM_MINOR}.${ROCM_VERSN}-1${RPM_TAG}.noarch.rpm ;; - 5.2 | 5.2.*) - ROCM_RPM=22.20${ROCM_SEP}${ROCM_PATCH}/rhel/${RPM_PATH}/amdgpu-install-22.20.${ROCM_VERSN}-1${RPM_TAG}.noarch.rpm - ;; - 5.1 | 5.1.*) - ROCM_RPM=22.10${ROCM_SEP}${ROCM_PATCH}/rhel/${RPM_PATH}/amdgpu-install-22.10${ROCM_SEP}${ROCM_PATCH}.${ROCM_VERSN}-1${RPM_TAG}.noarch.rpm - ;; - 5.0 | 5.0.*) - ROCM_RPM=21.50${ROCM_SEP}${ROCM_PATCH}/rhel/${RPM_PATH}/amdgpu-install-21.50${ROCM_SEP}${ROCM_PATCH}.${ROCM_VERSN}-1${RPM_TAG}.noarch.rpm - ;; - 4.5 | 4.5.*) - ROCM_RPM=21.40${ROCM_SEP}${ROCM_PATCH}/rhel/${RPM_PATH}/amdgpu-install-21.40${ROCM_SEP}${ROCM_PATCH}.${ROCM_VERSN}-1.noarch.rpm + 5.2 | 5.2.* | 5.1 | 5.1.* | 5.0 | 5.0.* | 4.*) + send-error "Invalid ROCm version ${ROCM_VERSION}. Supported: >= 5.3.0, <= 5.4.x" ;; 0.0) ;; @@ -227,7 +212,11 @@ do send-error "Unsupported combination :: ${DISTRO}-${VERSION} + ROCm ${ROCM_VERSION}" ;; esac - verbose-build docker build . -f ${DOCKER_FILE} --tag ${CONTAINER} --build-arg DISTRO=${DISTRO} --build-arg VERSION=${VERSION} --build-arg ROCM_VERSION=${ROCM_VERSION} --build-arg TOOLSET_VERSION=${TOOLSET_VERSION} --build-arg AMDGPU_RPM=${ROCM_RPM} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\" + + # use Rocky Linux as a base image for RHEL builds + DISTRO_BASE_IMAGE=rockylinux + + verbose-build docker build . -f ${DOCKER_FILE} --tag ${CONTAINER} --build-arg DISTRO=${DISTRO_BASE_IMAGE} --build-arg VERSION=${VERSION} --build-arg ROCM_VERSION=${ROCM_VERSION} --build-arg AMDGPU_RPM=${ROCM_RPM} --build-arg PYTHON_VERSIONS=\"${PYTHON_VERSIONS}\" elif [ "${DISTRO}" = "opensuse" ]; then case "${VERSION}" in 15.*) diff --git a/docker/entrypoint-centos.sh b/docker/entrypoint-rhel.sh similarity index 66% rename from docker/entrypoint-centos.sh rename to docker/entrypoint-rhel.sh index 12bb238ab8..c337ee8425 100755 --- a/docker/entrypoint-centos.sh +++ b/docker/entrypoint-rhel.sh @@ -1,11 +1,8 @@ #!/bin/bash -source scl_source enable devtoolset-9 source /etc/profile.d/modules.sh module load mpi -export LC_ALL=en_US.UTF-8 - if [ -z "${1}" ]; then exec bash else diff --git a/external/timemory b/external/timemory index 4b4b5445c2..8ca28b04d9 160000 --- a/external/timemory +++ b/external/timemory @@ -1 +1 @@ -Subproject commit 4b4b5445c2f421de19922b608bc80f7f7dd1849c +Subproject commit 8ca28b04d919f3fe89d4348fdd161b7876c4f852 diff --git a/scripts/test-install.sh b/scripts/test-install.sh index 68218e23e7..19052dc7af 100755 --- a/scripts/test-install.sh +++ b/scripts/test-install.sh @@ -177,17 +177,31 @@ test-omnitrace-python() test-omnitrace-rewrite() { - verbose-run omnitrace -e -v 1 -o ${CONFIG_DIR}/ls.inst --simulate -- ls + if [ -f /usr/bin/coreutils ]; then + local LS_NAME=coreutils + local LS_ARGS="--coreutils-prog=ls" + else + local LS_NAME=ls + local LS_ARGS="" + fi + verbose-run omnitrace -e -v 1 -o ${CONFIG_DIR}/ls.inst --simulate -- ${LS_NAME} for i in $(find ${CONFIG_DIR}/omnitrace-tests-output/ls.inst -type f); do verbose-run ls ${i}; done - verbose-run omnitrace -e -v 1 -o ${CONFIG_DIR}/ls.inst -- ls - verbose-run ${CONFIG_DIR}/ls.inst + verbose-run omnitrace -e -v 1 -o ${CONFIG_DIR}/ls.inst -- ${LS_NAME} + verbose-run ${CONFIG_DIR}/ls.inst ${LS_ARGS} } test-omnitrace-runtime() { - verbose-run omnitrace -e -v 1 --simulate -- ls - for i in $(find ${CONFIG_DIR}/omnitrace-tests-output/ls -type f); do verbose-run ls ${i}; done - verbose-run omnitrace -e -v 1 -- ls + if [ -f /usr/bin/coreutils ]; then + local LS_NAME=coreutils + local LS_ARGS="--coreutils-prog=ls" + else + local LS_NAME=ls + local LS_ARGS="" + fi + verbose-run omnitrace -e -v 1 --simulate -- ${LS_NAME} ${LS_ARGS} + for i in $(find ${CONFIG_DIR}/omnitrace-tests-output/$(basename ${LS_NAME}) -type f); do verbose-run ls ${i}; done + verbose-run omnitrace -e -v 1 -- ${LS_NAME} ${LS_ARGS} } test-omnitrace-critical-trace() diff --git a/source/bin/omnitrace/details.cpp b/source/bin/omnitrace/details.cpp index 24f29a5bb5..6cfe290cc8 100644 --- a/source/bin/omnitrace/details.cpp +++ b/source/bin/omnitrace/details.cpp @@ -27,7 +27,11 @@ #include #include +#include +#include +#include +#include #include #include @@ -43,10 +47,10 @@ get_whole_function_names() "sem_getvalue", "sem_clockwait", "sem_timedwait", "sem_trywait", "sem_unlink", "fork", "do_futex_wait", "dl_iterate_phdr", "dlinfo", "dlopen", "dlmopen", "dlvsym", "dlsym", "dlerror", "dladdr", "_dl_sym", "_dl_vsym", "_dl_addr", - "getenv", "setenv", "unsetenv", "printf", "fprintf", "vprintf", - "buffered_vfprintf", "vfprintf", "printf_positional", "puts", "fputs", "vfputs", - "fflush", "fwrite", "malloc", "malloc_stats", "malloc_trim", "mallopt", "calloc", - "free", "pvalloc", "valloc", "sysmalloc", "posix_memalign", "freehook", + "_dl_relocate_static_pie", "getenv", "setenv", "unsetenv", "printf", "fprintf", + "vprintf", "buffered_vfprintf", "vfprintf", "printf_positional", "puts", "fputs", + "vfputs", "fflush", "fwrite", "malloc", "malloc_stats", "malloc_trim", "mallopt", + "calloc", "free", "pvalloc", "valloc", "sysmalloc", "posix_memalign", "freehook", "mallochook", "memalignhook", "mprobe", "reallochook", "mmap", "munmap", "fopen", "fclose", "fmemopen", "fmemclose", "backtrace", "backtrace_symbols", "backtrace_symbols_fd", "sigaddset", "sigandset", "sigdelset", "sigemptyset", @@ -479,6 +483,198 @@ find_function(image_t* app_image, const std::string& _name, const strset_t& _ext return _func; } +//======================================================================================// +// +// Get the realpath to this exe +// +bool +is_text_file(const std::string& filename) +{ + std::ifstream _file{ filename, std::ios::in | std::ios::binary }; + if(!_file.is_open()) + { + errprintf(-1, "Error! '%s' could not be opened...\n", filename.c_str()); + return false; + } + + constexpr size_t buffer_size = 1024; + char buffer[buffer_size]; + while(_file.read(buffer, sizeof(buffer))) + { + for(char itr : buffer) + { + if(itr == '\0') return false; + } + } + + if(_file.gcount() > 0) + { + for(std::streamsize i = 0; i < _file.gcount(); ++i) + { + if(buffer[i] == '\0') return false; + } + } + + return true; +} + +//======================================================================================// +// +// Get the realpath to this exe +// +std::string& +omnitrace_get_exe_realpath() +{ + static std::string _v = []() { + auto _cmd_line = tim::read_command_line(tim::process::get_id()); + if(!_cmd_line.empty()) + { + using array_config_t = timemory::join::array_config; + OMNITRACE_ADD_DETAILED_LOG_ENTRY(array_config_t{ " ", "[ ", " ]" }, + "cmdline:: ", _cmd_line); + return _cmd_line.front(); + // return tim::filepath::realpath(_cmd_line.front(), nullptr, false); + } + return std::string{}; + }(); + return _v; +} + +//======================================================================================// +// +// Error callback routine. +// +std::vector +omnitrace_get_link_map(const char* _lib, const std::string& _exclude_linked_by, + const std::string& _exclude_re, std::vector&& _open_modes) +{ + if(_open_modes.empty()) _open_modes = { (RTLD_LAZY | RTLD_NOLOAD) }; + + auto _get_chain = [&_open_modes](const char* _name) { + void* _handle = nullptr; + bool _noload = false; + for(auto _mode : _open_modes) + { + _handle = dlopen(_name, _mode); + _noload = (_mode & RTLD_NOLOAD) == RTLD_NOLOAD; + if(_handle) break; + } + + auto _chain = std::vector{}; + if(_handle) + { + struct link_map* _link_map = nullptr; + dlinfo(_handle, RTLD_DI_LINKMAP, &_link_map); + struct link_map* _next = _link_map; + while(_next) + { + if(_name == nullptr && _next == _link_map && + std::string_view{ _next->l_name }.empty()) + { + // only insert exe name if dlopened the exe and + // empty name is first entry + _chain.emplace_back(omnitrace_get_exe_realpath()); + } + else if(!std::string_view{ _next->l_name }.empty()) + { + _chain.emplace_back(_next->l_name); + } + _next = _next->l_next; + } + + if(_noload == false) dlclose(_handle); + } + return _chain; + }; + + auto _full_chain = _get_chain(_lib); + auto _excl_chain = (_exclude_linked_by.empty()) + ? std::vector{} + : _get_chain(_exclude_linked_by.c_str()); + auto _fini_chain = std::vector{}; + _fini_chain.reserve(_full_chain.size()); + + for(const auto& itr : _full_chain) + { + auto _found = std::any_of(_excl_chain.begin(), _excl_chain.end(), + [itr](const auto& _v) { return (itr == _v); }); + if(!_found) + { + if(_exclude_re.empty() || !std::regex_search(itr, std::regex{ _exclude_re })) + _fini_chain.emplace_back(itr); + else + _excl_chain.emplace_back(itr); + } + } + + return _fini_chain; +} + +//======================================================================================// +// +// Get the path of a loaded dynamic binary +// +std::optional +omnitrace_get_loaded_path(const char* _name, std::vector&& _open_modes) +{ + if(_open_modes.empty()) _open_modes = { (RTLD_LAZY | RTLD_NOLOAD) }; + + void* _handle = nullptr; + bool _noload = false; + for(auto _mode : _open_modes) + { + _handle = dlopen(_name, _mode); + _noload = (_mode & RTLD_NOLOAD) == RTLD_NOLOAD; + if(_handle) break; + } + + if(_handle) + { + struct link_map* _link_map = nullptr; + dlinfo(_handle, RTLD_DI_LINKMAP, &_link_map); + if(_link_map != nullptr && !std::string_view{ _link_map->l_name }.empty()) + { + return tim::filepath::realpath(_link_map->l_name, nullptr, false); + } + if(_noload == false) dlclose(_handle); + } + + return std::optional{}; +} + +//======================================================================================// +// +// Get the path of a loaded dynamic binary +// +std::optional +omnitrace_get_origin(const char* _name, std::vector&& _open_modes) +{ + if(_open_modes.empty()) _open_modes = { (RTLD_LAZY | RTLD_NOLOAD) }; + + void* _handle = nullptr; + bool _noload = false; + for(auto _mode : _open_modes) + { + _handle = dlopen(_name, _mode); + _noload = (_mode & RTLD_NOLOAD) == RTLD_NOLOAD; + if(_handle) break; + } + + if(_handle) + { + char _buffer[PATH_MAX + 1]; + memset(_buffer, '\0', PATH_MAX * sizeof(char)); + dlinfo(_handle, RTLD_DI_ORIGIN, _buffer); + if(strnlen(_buffer, PATH_MAX + 1) <= PATH_MAX) + { + return tim::filepath::realpath(_buffer, nullptr, false); + } + if(_noload == false) dlclose(_handle); + } + + return std::optional{}; +} + //======================================================================================// // // Error callback routine. diff --git a/source/bin/omnitrace/log.hpp b/source/bin/omnitrace/log.hpp index 0cfd53e1a6..beb19528cc 100644 --- a/source/bin/omnitrace/log.hpp +++ b/source/bin/omnitrace/log.hpp @@ -89,4 +89,4 @@ private: #define OMNITRACE_ADD_DETAILED_LOG_ENTRY(DELIM, ...) \ log_entry::add_log_entry( \ { log_entry::source_location{ __FUNCTION__, __FILE__, __LINE__ }, \ - timemory::join::join(__VA_ARGS__) }) + timemory::join::join(DELIM, __VA_ARGS__) }) diff --git a/source/bin/omnitrace/omnitrace.cpp b/source/bin/omnitrace/omnitrace.cpp index 417faba6ff..1ba2717aa6 100644 --- a/source/bin/omnitrace/omnitrace.cpp +++ b/source/bin/omnitrace/omnitrace.cpp @@ -28,13 +28,16 @@ #include #include +#include #include #include #include #include #include #include +#include #include +#include #include #include @@ -55,6 +58,7 @@ #include #include #include +#include #include #include @@ -82,28 +86,29 @@ get_default_min_address_range() } } // namespace -bool use_return_info = false; -bool use_args_info = false; -bool use_file_info = false; -bool use_line_info = false; -bool allow_overlapping = false; -bool loop_level_instr = false; -bool instr_dynamic_callsites = false; -bool instr_traps = false; -bool instr_loop_traps = false; -bool parse_all_modules = false; -size_t min_address_range = get_default_min_address_range(); // 4096 -size_t min_loop_address_range = get_default_min_address_range(); // 4096 -size_t min_instructions = get_default_min_instructions(); // 1024 -size_t min_loop_instructions = get_default_min_instructions(); // 1024 -bool werror = false; -bool debug_print = false; -bool instr_print = false; -bool simulate = false; -bool include_uninstr = false; -bool include_internal_linked_libs = false; -int verbose_level = tim::get_env("OMNITRACE_VERBOSE_INSTRUMENT", 0); -int num_log_entries = tim::get_env("OMNITRACE_LOG_COUNT", 20); +bool use_return_info = false; +bool use_args_info = false; +bool use_file_info = false; +bool use_line_info = false; +bool allow_overlapping = false; +bool loop_level_instr = false; +bool instr_dynamic_callsites = false; +bool instr_traps = false; +bool instr_loop_traps = false; +bool parse_all_modules = false; +size_t min_address_range = get_default_min_address_range(); // 4096 +size_t min_loop_address_range = get_default_min_address_range(); // 4096 +size_t min_instructions = get_default_min_instructions(); // 1024 +size_t min_loop_instructions = get_default_min_instructions(); // 1024 +bool werror = false; +bool debug_print = false; +bool instr_print = false; +bool simulate = false; +bool include_uninstr = false; +bool include_internal_linked_libs = false; +int verbose_level = tim::get_env("OMNITRACE_VERBOSE_INSTRUMENT", 0); +int num_log_entries = tim::get_env( + "OMNITRACE_LOG_COUNT", tim::get_env("OMNITRACE_CI", false) ? 20 : -1); string_t main_fname = "main"; string_t argv0 = {}; string_t cmdv0 = {}; @@ -143,8 +148,9 @@ std::unique_ptr log_ofs = {}; namespace { -namespace process = tim::process; -namespace signals = tim::signals; +namespace process = tim::process; // NOLINT +namespace signals = tim::signals; +namespace filepath = tim::filepath; using signal_settings = tim::signals::signal_settings; using sys_signal = tim::signals::sys_signal; @@ -175,11 +181,16 @@ string_t print_excluded = {}; string_t print_available = {}; string_t print_overlapping = {}; strset_t print_formats = { "txt", "json" }; -strvec_t libname_suffixes = {}; -strvec_t libname_fallbacks = {}; std::string modfunc_dump_dir = {}; auto regex_opts = std::regex_constants::egrep | std::regex_constants::optimize; +strvec_t lib_search_paths = + tim::delimit(JOIN(':', tim::get_env("DYNINSTAPI_RT_LIB"), + tim::get_env("DYNINST_REWRITER_PATHS"), + tim::get_env("LD_LIBRARY_PATH")), + ":"); +strvec_t bin_search_paths = tim::delimit(tim::get_env("PATH"), ":"); + #if defined(DYNINST_API_RT) auto _dyn_api_rt_paths = tim::delimit(DYNINST_API_RT, ":"); #else @@ -187,16 +198,25 @@ auto _dyn_api_rt_paths = std::vector{}; #endif std::string -get_absolute_exe_filepath(std::string exe_name, const std::string& env_path = "PATH"); +get_absolute_filepath(std::string _name, const strvec_t& _paths); std::string -get_absolute_lib_filepath(std::string lib_name, - const std::string& env_path = "LD_LIBRARY_PATH", - std::vector suffixes = {}, - std::vector fallbacks = {}); +get_absolute_filepath(std::string _name); + +std::string +get_absolute_exe_filepath(std::string exe_name); + +std::string +get_absolute_lib_filepath(std::string lib_name); bool -file_exists(const std::string& name); +exists(const std::string& name); + +bool +is_file(std::string _name); + +bool +is_directory(std::string _name); std::string get_realpath(const std::string&); @@ -259,6 +279,12 @@ auto _activate = sys_signal::FileSize, sys_signal::CPUtime }), true); +auto +find(const std::string& itr, const strvec_t& _data) +{ + return std::any_of(_data.begin(), _data.end(), + [itr](const auto& _v) { return (itr == _v); }); +} } // namespace //======================================================================================// @@ -272,7 +298,67 @@ main(int argc, char** argv) { argv0 = argv[0]; - OMNITRACE_ADD_LOG_ENTRY(argv[0]); + auto _omni_root = tim::get_env( + "omnitrace_ROOT", tim::get_env("OMNITRACE_ROOT", "")); + if(!_omni_root.empty() && exists(_omni_root)) + { + bin_search_paths.emplace_back(JOIN('/', _omni_root, "bin")); + bin_search_paths.emplace_back(JOIN('/', _omni_root, "lib", "omnitrace")); + bin_search_paths.emplace_back(JOIN('/', _omni_root, "lib", "omnitrace", "bin")); + lib_search_paths.emplace_back(JOIN('/', _omni_root, "lib")); + lib_search_paths.emplace_back(JOIN('/', _omni_root, "lib", "omnitrace")); + lib_search_paths.emplace_back(JOIN('/', _omni_root, "lib", "omnitrace", "lib")); + lib_search_paths.emplace_back(JOIN('/', _omni_root, "lib", "omnitrace", "lib64")); + OMNITRACE_ADD_LOG_ENTRY(argv[0], "::", "omnitrace root path: ", _omni_root); + } + + auto _omni_exe_path = get_realpath(get_absolute_exe_filepath(argv[0])); + if(!exists(_omni_exe_path)) + _omni_exe_path = + get_realpath(get_absolute_exe_filepath(omnitrace_get_exe_realpath())); + bin_search_paths.emplace_back(filepath::dirname(_omni_exe_path)); + + auto _omni_lib_path = + JOIN('/', filepath::dirname(filepath::dirname(_omni_exe_path)), "lib"); + bin_search_paths.emplace_back(JOIN('/', _omni_lib_path, "omnitrace")); + bin_search_paths.emplace_back(JOIN('/', _omni_lib_path, "omnitrace", "bin")); + lib_search_paths.emplace_back(_omni_lib_path); + lib_search_paths.emplace_back(JOIN('/', _omni_lib_path, "omnitrace")); + lib_search_paths.emplace_back(JOIN('/', _omni_lib_path, "omnitrace", "lib")); + lib_search_paths.emplace_back(JOIN('/', _omni_lib_path, "omnitrace", "lib64")); + + OMNITRACE_ADD_LOG_ENTRY(argv[0], "::", "omnitrace bin path: ", _omni_exe_path); + OMNITRACE_ADD_LOG_ENTRY(argv[0], "::", "omnitrace lib path: ", _omni_lib_path); + + for(const auto& itr : omnitrace_get_link_map(nullptr)) + { + if(itr.find("omnitrace") != std::string::npos || + std::regex_search( + itr, std::regex{ "lib(dyninstAPI|stackwalk|pcontrol|patchAPI|parseAPI|" + "instructionAPI|symtabAPI|dynDwarf|common|dynElf|tbb|" + "tbbmalloc|tbbmalloc_proxy|gotcha|libunwind|roctracer|" + "hsa-runtime|amdhip|rocm_smi)\\.(so|a)" })) + { + if(!find(filepath::dirname(itr), lib_search_paths)) + lib_search_paths.emplace_back(filepath::dirname(itr)); + } + } + + // DO NOT SORT! Just remove adjacent duplicates + bin_search_paths.erase(std::unique(bin_search_paths.begin(), bin_search_paths.end()), + bin_search_paths.end()); + lib_search_paths.erase(std::unique(lib_search_paths.begin(), lib_search_paths.end()), + lib_search_paths.end()); + + for(const auto& itr : bin_search_paths) + { + OMNITRACE_ADD_LOG_ENTRY("bin search path:", itr); + } + + for(const auto& itr : lib_search_paths) + { + OMNITRACE_ADD_LOG_ENTRY("lib search path:", itr); + } address_space_t* addr_space = nullptr; string_t mutname = {}; @@ -343,7 +429,7 @@ main(int argc, char** argv) if(_cmdc > 0 && !mutname.empty()) { - auto resolved_mutname = get_absolute_exe_filepath(mutname); + auto resolved_mutname = get_realpath(get_absolute_filepath(mutname)); if(resolved_mutname != mutname) { mutname = resolved_mutname; @@ -550,7 +636,7 @@ main(int argc, char** argv) p.print_help(extra_help); std::exit(EXIT_FAILURE); } - keys.at(0) = get_absolute_exe_filepath(keys.at(0)); + keys.at(0) = get_realpath(get_absolute_filepath(keys.at(0))); mutname = keys.at(0); _cmdc = keys.size(); _cmdv = new char*[_cmdc]; @@ -1079,28 +1165,6 @@ main(int argc, char** argv) auto _omnitrace_exe_path = tim::dirname(::get_realpath("/proc/self/exe")); verbprintf(4, "omnitrace exe path: %s\n", _omnitrace_exe_path.c_str()); - if(strcmp(::basename(_omnitrace_exe_path.c_str()), "bin") == 0) - { - libname_suffixes.emplace_back("omnitrace"); - libname_suffixes.emplace_back("../lib"); - libname_suffixes.emplace_back("../lib64"); - libname_suffixes.emplace_back("../lib/omnitrace"); - libname_suffixes.emplace_back("../lib64/omnitrace"); - libname_suffixes.emplace_back("lib"); - libname_suffixes.emplace_back("lib64"); - libname_suffixes.emplace_back("lib/omnitrace"); - libname_suffixes.emplace_back("lib64/omnitrace"); - libname_fallbacks.emplace_back(_omnitrace_exe_path); - } - else - { - libname_suffixes.emplace_back("lib"); - libname_suffixes.emplace_back("lib64"); - libname_suffixes.emplace_back("lib/omnitrace"); - libname_suffixes.emplace_back("lib64/omnitrace"); - libname_fallbacks.emplace_back(tim::get_env("PWD", ".")); - } - if(_cmdv && _cmdv[0] && strlen(_cmdv[0]) > 0) { auto _is_executable = omnitrace_get_is_executable(_cmdv[0], binary_rewrite); @@ -1190,7 +1254,7 @@ main(int argc, char** argv) log_ofs = std::make_unique(); verbprintf_bare(0, "%s", ::tim::log::color::source()); verbprintf(0, "Opening '%s' for log output... ", logfile.c_str()); - if(!tim::filepath::open(*log_ofs, logfile)) + if(!filepath::open(*log_ofs, logfile)) throw std::runtime_error(JOIN(" ", "Error opening log output file", logfile)); verbprintf_bare(0, "Done\n%s", ::tim::log::color::end()); print_log_entries(*log_ofs, -1, {}, {}, "", false); @@ -1261,6 +1325,12 @@ main(int argc, char** argv) // //----------------------------------------------------------------------------------// + for(const auto& itr : _dyn_api_rt_paths) + { + lib_search_paths.emplace_back(itr); + lib_search_paths.emplace_back(filepath::dirname(itr)); + } + find_dyn_api_rt(); int dyninst_verb = 2; @@ -1548,7 +1618,7 @@ main(int argc, char** argv) for(auto _libname : _libnames) { OMNITRACE_ADD_LOG_ENTRY("Getting the absolute lib filepath to", _libname); - _libname = get_absolute_lib_filepath(_libname); + _libname = get_realpath(get_absolute_lib_filepath(_libname)); _tried_libs += string_t("|") + _libname; verbprintf(1, "loading library: '%s'...\n", _libname.c_str()); result = (addr_space->loadLibrary(_libname.c_str()) != nullptr); @@ -1846,13 +1916,11 @@ main(int argc, char** argv) std::string _libname = {}; for(auto&& itr : sharedlibname) { - if(_libname.empty()) _libname = get_absolute_lib_filepath(itr, "LD_LIBRARY_PATH"); - if(_libname.empty()) _libname = get_absolute_lib_filepath(itr, "LIBRARY_PATH"); + if(_libname.empty()) _libname = get_absolute_lib_filepath(itr); } for(auto&& itr : staticlibname) { - if(_libname.empty()) _libname = get_absolute_lib_filepath(itr, "LIBRARY_PATH"); - if(_libname.empty()) _libname = get_absolute_lib_filepath(itr, "LD_LIBRARY_PATH"); + if(_libname.empty()) _libname = get_absolute_lib_filepath(itr); } if(_libname.empty()) _libname = "libomnitrace-dl.so"; @@ -2531,118 +2599,181 @@ namespace //======================================================================================// // std::string -get_absolute_exe_filepath(std::string exe_name, const std::string& env_path) +canonicalize(std::string _path) { - if(!exe_name.empty() && !file_exists(exe_name)) - { - auto _exe_orig = exe_name; - auto _paths = tim::delimit(tim::get_env(env_path, ""), ":"); - for(auto& pitr : _paths) - { - if(file_exists(TIMEMORY_JOIN('/', pitr, exe_name))) - { - exe_name = get_realpath(TIMEMORY_JOIN('/', pitr, exe_name)); - verbprintf(1, "Resolved '%s' to '%s'...\n", _exe_orig.c_str(), - exe_name.c_str()); - break; - } - } + if(_path.find("./") == 0) + _path = _path.replace(0, 1, get_cwd()); + else if(_path.find("../") == 0) + _path = _path.insert(0, get_cwd() + "/"); - if(!file_exists(exe_name)) - { - verbprintf(0, "Warning! File path to '%s' could not be determined...\n", - exe_name.c_str()); - } - } - else if(!exe_name.empty()) + auto _leading_dash = (_path.find('/') == 0); + auto _pieces = tim::delimit(_path, "/"); + std::reverse(_pieces.begin(), _pieces.end()); + auto _tree = std::vector{}; + for(size_t i = 0; i < _pieces.size(); ++i) { - return get_realpath(exe_name); + const auto& itr = _pieces.at(i); + if(itr == ".") + { + continue; + } + else if(itr == "..") + ++i; + else + _tree.emplace_back(itr); } - - return exe_name; + std::reverse(_tree.begin(), _tree.end()); + auto _cpath = std::string{ (_leading_dash) ? "/" : "" }; + for(size_t i = 0; i < _tree.size() - 1; ++i) + _cpath += _tree.at(i) + "/"; + _cpath += _tree.back(); + return _cpath; } //======================================================================================// // std::string -get_absolute_lib_filepath(std::string lib_name, const std::string& env_path, - std::vector suffixes, - std::vector fallbacks) +absolute(std::string _path) { - if(suffixes.empty()) suffixes = libname_suffixes; - if(fallbacks.empty()) fallbacks = libname_fallbacks; - - if(!lib_name.empty() && (!file_exists(lib_name) || - std::regex_match(lib_name, std::regex("^[A-Za-z0-9].*")))) - { - auto _lib_orig = lib_name; - auto _paths = tim::delimit( - std::string{ ".:" } + tim::get_env(env_path, ""), ":"); - std::copy(fallbacks.begin(), fallbacks.end(), std::back_inserter(_paths)); - for(auto& pitr : _paths) - { - if(file_exists(TIMEMORY_JOIN('/', pitr, lib_name))) - { - lib_name = get_realpath(TIMEMORY_JOIN('/', pitr, lib_name)); - verbprintf(1, "Resolved '%s' to '%s'...\n", _lib_orig.c_str(), - lib_name.c_str()); - break; - } - for(auto& sitr : suffixes) - { - if(sitr.empty()) continue; - if(file_exists(TIMEMORY_JOIN('/', pitr, sitr, lib_name))) - { - lib_name = get_realpath(TIMEMORY_JOIN('/', pitr, sitr, lib_name)); - verbprintf(1, "Resolved '%s' to '%s'...\n", _lib_orig.c_str(), - lib_name.c_str()); - break; - } - } - if(file_exists(lib_name)) break; - } - - if(!file_exists(lib_name)) - { - verbprintf(0, "Warning! File path to '%s' could not be determined...\n", - lib_name.c_str()); - } - } - else if(!lib_name.empty()) - { - return get_realpath(lib_name); - } - - return lib_name; + if(_path.find('/') == 0) return canonicalize(_path); + return canonicalize(JOIN('/', get_cwd(), _path)); } //======================================================================================// // -bool -file_exists(const std::string& name) +std::string +get_absolute_filepath(std::string _name, const strvec_t& _search_paths) { + if(!_name.empty() && (!exists(_name) || !is_file(_name))) + { + auto _orig = _name; + for(auto itr : _search_paths) + { + if(!is_directory(itr) || is_file(itr)) itr = filepath::dirname(itr); + + auto _exists = false; + OMNITRACE_ADD_LOG_ENTRY("searching", itr, "for", _name); + for(const auto& pitr : + { absolute(JOIN('/', itr, _name)), + absolute(JOIN('/', itr, filepath::basename(_name))) }) + { + _exists = exists(pitr) && is_file(pitr); + if(_exists) + { + _name = pitr; + verbprintf(1, "Resolved '%s' to '%s'...\n", _orig.c_str(), + _name.c_str()); + break; + } + } + if(_exists) break; + } + + if(!exists(_name)) + { + using array_config_t = timemory::join::array_config; + auto _search_paths_v = + timemory::join::join(array_config_t{ ", ", "", "" }, bin_search_paths); + verbprintf( + 0, "Warning! File path to '%s' could not be determined... search: %s\n", + _name.c_str(), _search_paths_v.c_str()); + } + } + else if(!_name.empty()) + { + auto _orig = _name; + _name = absolute(_name); + verbprintf(1, "Resolved '%s' to '%s'...\n", _orig.c_str(), _name.c_str()); + } + + return _name; +} + +//======================================================================================// +// +std::string +get_absolute_filepath(std::string _name) +{ + auto _search_paths = strvec_t{}; + auto _combine_paths = std::vector{ bin_search_paths, lib_search_paths }; + auto _base_name = std::string_view{ filepath::basename(_name) }; + // if the name looks like a library, put the lib_search_paths first + if(_base_name.find("lib") == 0 || _base_name.find(".so") != std::string::npos || + _base_name.find(".a") != std::string::npos) + std::reverse(_combine_paths.begin(), _combine_paths.end()); + _search_paths.reserve(bin_search_paths.size() + lib_search_paths.size()); + for(const auto& pitr : _combine_paths) + for(const auto& itr : pitr) + _search_paths.emplace_back(itr); + + return get_absolute_filepath(std::move(_name), _search_paths); +} + +//======================================================================================// +// +std::string +get_absolute_exe_filepath(std::string exe_name) +{ + return get_absolute_filepath(std::move(exe_name), bin_search_paths); +} + +//======================================================================================// +// +std::string +get_absolute_lib_filepath(std::string lib_name) +{ + auto _orig_name = lib_name; + lib_name = get_absolute_filepath(std::move(lib_name), lib_search_paths); + if(_orig_name == lib_name && !exists(lib_name) && + lib_name.find(".so") == std::string::npos && + lib_name.find(".a") == std::string::npos) + { + lib_name = get_absolute_filepath(lib_name + ".so", lib_search_paths); + } + return lib_name; +} + +bool +exists(const std::string& name) +{ + return filepath::exists(absolute(name)); +} + +bool +is_file(std::string _name) +{ + _name = get_realpath(_name); struct stat buffer; - verbprintf(4, "querying whether file '%s' exists...\n", name.c_str()); - return (stat(name.c_str(), &buffer) == 0); + return (stat(_name.c_str(), &buffer) == 0 && S_ISREG(buffer.st_mode) != 0); +} + +bool +is_directory(std::string _name) +{ + _name = get_realpath(_name); + struct stat buffer; + return (stat(_name.c_str(), &buffer) == 0 && S_ISDIR(buffer.st_mode) != 0); } std::string get_realpath(const std::string& _f) { - char _buffer[PATH_MAX]; - if(!::realpath(_f.c_str(), _buffer)) - { - verbprintf(2, "Warning! realpath could not be found for %s\n", _f.c_str()); - return _f; - } - return std::string{ _buffer }; + return filepath::realpath(_f, nullptr, false); } std::string get_cwd() { - char cwd[PATH_MAX]; - return std::string{ getcwd(cwd, PATH_MAX) }; +#if defined(__GNUC__) + auto* _cwd_c = getcwd(nullptr, 0); + auto _cwd_v = std::string{ _cwd_c }; + free(_cwd_c); + return _cwd_v; +#else + char cwd[PATH_MAX]; + auto* _cwd_v = getcwd(cwd, PATH_MAX); + return (_cwd_v) ? std::string{ _cwd_v } : get_env("PWD"); +#endif } using tim::dirname; @@ -2650,12 +2781,6 @@ using tim::dirname; void find_dyn_api_rt() { - strvec_t _suffixes = libname_suffixes; - strvec_t _fallbacks = libname_fallbacks; - - std::copy(_dyn_api_rt_paths.begin(), _dyn_api_rt_paths.end(), - std::back_inserter(_fallbacks)); - #if defined(OMNITRACE_BUILD_DYNINST) std::string _dyn_api_rt_base = (binary_rewrite) ? "libomnitrace-rt" : "libdyninstAPI_RT"; @@ -2663,62 +2788,23 @@ find_dyn_api_rt() std::string _dyn_api_rt_base = "libdyninstAPI_RT"; #endif - auto _dyn_api_rt_abs = get_absolute_lib_filepath( - _dyn_api_rt_base + ".so", "LD_LIBRARY_PATH", _suffixes, _fallbacks); + auto _dyn_api_rt_env = + tim::get_env("DYNINSTAPI_RT_LIB", _dyn_api_rt_base + ".so"); + auto _dyn_api_rt_abs = get_absolute_lib_filepath(_dyn_api_rt_env); - if(_dyn_api_rt_abs != _dyn_api_rt_base + ".so") - _dyn_api_rt_paths.insert(_dyn_api_rt_paths.begin(), _dyn_api_rt_abs); - else + if(!exists(_dyn_api_rt_abs)) + _dyn_api_rt_abs = get_absolute_lib_filepath(_dyn_api_rt_base + ".a"); + + if(exists(_dyn_api_rt_abs)) { - _dyn_api_rt_abs = get_absolute_lib_filepath( - _dyn_api_rt_base + ".a", "LIBRARY_PATH", _suffixes, _fallbacks); - if(_dyn_api_rt_abs != _dyn_api_rt_base + ".a") - _dyn_api_rt_paths.insert(_dyn_api_rt_paths.begin(), _dyn_api_rt_abs); + namespace join = ::timemory::join; + tim::set_env("DYNINSTAPI_RT_LIB", _dyn_api_rt_abs, 1); + tim::set_env("DYNINST_REWRITER_PATHS", + join::join(join::array_config{ ":", "", "" }, + dirname(_dyn_api_rt_abs), lib_search_paths), + 1); } - auto _rewriter_paths = tim::get_env("DYNINST_REWRITER_PATHS", ""); - for(auto itr : _dyn_api_rt_paths) - { - auto _file_exists = [](const std::string& _fname) { - struct stat _buffer; - if(stat(_fname.c_str(), &_buffer) == 0) - return (S_ISREG(_buffer.st_mode) != 0 || S_ISLNK(_buffer.st_mode) != 0); - return false; - }; - - auto _export = [&_rewriter_paths, &_file_exists](std::string _fname) { - int _overwrite = - (_file_exists(tim::get_env("DYNINSTAPI_RT_LIB", ""))) ? 0 : 1; - tim::set_env("DYNINSTAPI_RT_LIB", _fname, _overwrite); - _fname = tim::get_env("DYNINSTAPI_RT_LIB", _fname); - tim::set_env( - "DYNINST_REWRITER_PATHS", - _rewriter_paths.empty() - ? dirname(_fname) - : TIMEMORY_JOIN(':', dirname(_fname), _rewriter_paths), - 1); - }; - - auto _resolved = [&](std::string _fname) { - if(_file_exists(_fname)) - { - _export(_fname); - return true; - } - _fname = get_absolute_lib_filepath(_fname, "LD_LIBRARY_PATH", _suffixes, - _fallbacks); - if(_file_exists(_fname)) - { - _export(_fname); - return true; - } - return false; - }; - - if(_resolved(itr)) break; - if(_resolved(TIMEMORY_JOIN('/', itr, _dyn_api_rt_base + ".so"))) break; - if(_resolved(TIMEMORY_JOIN('/', itr, _dyn_api_rt_base + ".a"))) break; - } auto _v = tim::get_env("DYNINSTAPI_RT_LIB", ""); verbprintf(0, "DYNINST_API_RT: %s\n", (_v.empty()) ? "" : _v.c_str()); } diff --git a/source/bin/omnitrace/omnitrace.hpp b/source/bin/omnitrace/omnitrace.hpp index dd71d4a806..09cf707f83 100644 --- a/source/bin/omnitrace/omnitrace.hpp +++ b/source/bin/omnitrace/omnitrace.hpp @@ -30,34 +30,16 @@ #include +#include +#include #include #include #include //======================================================================================// -inline string_t -get_absolute_path(const char* fname) -{ - char path_save[PATH_MAX]; - char abs_exe_path[PATH_MAX]; - char* p = nullptr; - - if(!(p = strrchr((char*) fname, '/'))) - { - auto* ret = getcwd(abs_exe_path, sizeof(abs_exe_path)); - consume_parameters(ret); - } - else - { - auto* rets = getcwd(path_save, sizeof(path_save)); - auto retf = chdir(fname); - auto* reta = getcwd(abs_exe_path, sizeof(abs_exe_path)); - auto retp = chdir(path_save); - consume_parameters(rets, retf, reta, retp); - } - return string_t(abs_exe_path); -} +bool +is_text_file(const std::string& filename); //======================================================================================// @@ -190,12 +172,20 @@ omnitrace_get_is_executable(std::string_view _cmd, bool _default_v) // static inline address_space_t* omnitrace_get_address_space(patch_pointer_t& _bpatch, int _cmdc, char** _cmdv, - bool _rewrite, int _pid = -1, const string_t& _name = {}) + bool _rewrite, int _pid = -1, const std::string& _name = {}) { address_space_t* mutatee = nullptr; if(_rewrite) { + if(is_text_file(_name)) + { + errprintf( + -127, + "'%s' is a text file. OmniTrace only supports instrumenting binary files", + _name.c_str()); + } + verbprintf(1, "Opening '%s' for binary rewrite... ", _name.c_str()); fflush(stderr); if(!_name.empty()) mutatee = _bpatch->openBinary(_name.c_str(), false); @@ -221,6 +211,16 @@ omnitrace_get_address_space(patch_pointer_t& _bpatch, int _cmdc, char** _cmdv, } else { + if(_cmdc < 1) errprintf(-127, "No command provided"); + + if(is_text_file(_cmdv[0])) + { + errprintf(-1, + "'%s' is a text file. OmniTrace only supports instrumenting " + "binary files", + _cmdv[0]); + } + std::stringstream ss; for(int i = 0; i < _cmdc; ++i) { @@ -323,6 +323,21 @@ omnitrace_fork_callback(thread_t* parent, thread_t* child) } // //======================================================================================// +// path resolution helpers +// +std::string& +omnitrace_get_exe_realpath(); +// +std::optional +omnitrace_get_origin(const char* _name, + std::vector&& _open_modes = { (RTLD_LAZY | RTLD_NOLOAD) }); +// +std::vector +omnitrace_get_link_map(const char* _lib, const std::string& _exclude_linked_by = {}, + const std::string& _exclude_re = {}, + std::vector&& _open_modes = { (RTLD_LAZY | RTLD_NOLOAD) }); +// +//======================================================================================// // insert_instr -- insert instrumentation into a function // template diff --git a/source/bin/tests/CMakeLists.txt b/source/bin/tests/CMakeLists.txt index df23b718a0..59aecaa0e1 100644 --- a/source/bin/tests/CMakeLists.txt +++ b/source/bin/tests/CMakeLists.txt @@ -34,7 +34,8 @@ function(OMNITRACE_ADD_BIN_TEST) # common list(APPEND TEST_ENVIRONMENT "OMNITRACE_CI=ON" "OMNITRACE_CONFIG_FILE=" - "OMNITRACE_OUTPUT_PATH=omnitrace-tests-output" "TWD=${TEST_WORKING_DIRECTORY}") + "OMNITRACE_OUTPUT_PATH=${PROJECT_BINARY_DIR}/omnitrace-tests-output" + "TWD=${TEST_WORKING_DIRECTORY}") # copy for inverse set(TEST_ENVIRONMENT_INV "${TEST_ENVIRONMENT}") @@ -155,6 +156,15 @@ omnitrace_add_bin_test( ".*\\\[omnitrace\\\] Usage:.*\\\[DEBUG OPTIONS\\\].*\\\[MODE OPTIONS\\\].*\\\[LIBRARY OPTIONS\\\].*\\\[SYMBOL SELECTION OPTIONS\\\].*\\\[RUNTIME OPTIONS\\\].*\\\[GRANULARITY OPTIONS\\\].*\\\[DYNINST OPTIONS\\\].*" ) +# on RedHat, /usr/bin/ls is a script for `coreutils --coreutils-prog=ls` +if(EXISTS /usr/bin/coreutils) + set(LS_NAME "coreutils") + set(LS_ARGS "--coreutils-prog=ls") +else() + set(LS_NAME ls) + set(LS_ARGS) +endif() + omnitrace_add_bin_test( NAME omnitrace-exe-simulate-ls TARGET omnitrace-exe @@ -167,9 +177,10 @@ omnitrace_add_bin_test( 2 --all-functions -- - ls + ${LS_NAME} + ${LS_ARGS} LABELS "simulate" - TIMEOUT 120) + TIMEOUT 240) omnitrace_add_bin_test( NAME omnitrace-exe-simulate-ls-check @@ -191,10 +202,35 @@ omnitrace_add_bin_test( "\\\[omnitrace\\\]\\\[exe\\\] Runtime instrumentation is not possible!(.*)\n(.*)\\\[omnitrace\\\]\\\[exe\\\] Switching to binary rewrite mode and assuming '--simulate --all-functions'" ) +file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/omnitrace-tests-output/tmp) + +omnitrace_add_bin_test( + NAME omnitrace-exe-simulate-lib-basename + TARGET omnitrace-exe + ARGS --print-available + functions + -v + 2 + -o + ${PROJECT_BINARY_DIR}/omnitrace-tests-output/omnitrace-exe-simulate-lib-basename/${CMAKE_SHARED_LIBRARY_PREFIX}$${CMAKE_SHARED_LIBRARY_SUFFIX} + -- + ${CMAKE_SHARED_LIBRARY_PREFIX}$${CMAKE_SHARED_LIBRARY_SUFFIX} + LABELS "simulate" + TIMEOUT 120 + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/omnitrace-tests-output/tmp) + omnitrace_add_bin_test( NAME omnitrace-exe-write-log TARGET omnitrace-exe - ARGS --print-instrumented functions -v 1 --log-file user.log -- ls + ARGS --print-instrumented + functions + -v + 1 + --log-file + user.log + -- + ${LS_NAME} + ${LS_ARGS} LABELS "log" TIMEOUT 120 PASS_REGEX "Opening .*/instrumentation/user.log") diff --git a/source/lib/omnitrace/library.cpp b/source/lib/omnitrace/library.cpp index 1986bce0c5..e0e5f2fc26 100644 --- a/source/lib/omnitrace/library.cpp +++ b/source/lib/omnitrace/library.cpp @@ -180,6 +180,8 @@ ensure_finalization(bool _static_init = false) OMNITRACE_DEBUG_F("\n"); } + if(_timemory_manager) _timemory_manager->set_write_metadata(-1); + return scope::destructor{ []() { omnitrace_finalize_hidden(); } }; } @@ -993,8 +995,6 @@ omnitrace_finalize_hidden(void) tim::cereal::make_nvp("memory_maps", _maps)); }); - _timemory_manager->set_write_metadata(-1); - OMNITRACE_VERBOSE_F(1, "Finalizing timemory...\n"); tim::timemory_finalize(_timemory_manager.get()); diff --git a/source/lib/omnitrace/library/causal/components/backtrace.cpp b/source/lib/omnitrace/library/causal/components/backtrace.cpp index ef50dd9c5b..f1d472a506 100644 --- a/source/lib/omnitrace/library/causal/components/backtrace.cpp +++ b/source/lib/omnitrace/library/causal/components/backtrace.cpp @@ -117,6 +117,8 @@ backtrace::sample(int _sig) } ++_protect_flag; + // on RedHat, the unw_step within get_unw_signal_frame_stack_raw involves a mutex lock + OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal); m_index = causal::experiment::get_index(); m_stack = get_unw_signal_frame_stack_raw(); diff --git a/source/lib/omnitrace/library/components/backtrace.cpp b/source/lib/omnitrace/library/components/backtrace.cpp index c9d65dfe8c..2df0f10f4a 100644 --- a/source/lib/omnitrace/library/components/backtrace.cpp +++ b/source/lib/omnitrace/library/components/backtrace.cpp @@ -25,6 +25,7 @@ #include "core/config.hpp" #include "core/debug.hpp" #include "core/perfetto.hpp" +#include "core/state.hpp" #include "library/components/ensure_storage.hpp" #include "library/ptl.hpp" #include "library/runtime.hpp" @@ -184,6 +185,9 @@ backtrace::size() const void backtrace::sample(int) { + // on RedHat, the unw_step within get_unw_stack involves a mutex lock + OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal); + using namespace tim::backtrace; constexpr bool with_signal_frame = false; constexpr size_t ignore_depth = 3; diff --git a/source/lib/omnitrace/library/components/mpi_gotcha.cpp b/source/lib/omnitrace/library/components/mpi_gotcha.cpp index ed48ee9053..5e552e91e9 100644 --- a/source/lib/omnitrace/library/components/mpi_gotcha.cpp +++ b/source/lib/omnitrace/library/components/mpi_gotcha.cpp @@ -128,10 +128,17 @@ void omnitrace_mpi_set_attr() { #if defined(TIMEMORY_USE_MPI) + auto _blocked = get_sampling_signals(); + if(!_blocked.empty()) + tim::signals::block_signals(_blocked, tim::signals::sigmask_scope::process); + int _comm_key = -1; if(PMPI_Comm_create_keyval(&omnitrace_mpi_copy, &omnitrace_mpi_fini, &_comm_key, nullptr) == MPI_SUCCESS) PMPI_Comm_set_attr(MPI_COMM_SELF, _comm_key, nullptr); + + if(!_blocked.empty()) + tim::signals::unblock_signals(_blocked, tim::signals::sigmask_scope::process); #endif } @@ -263,7 +270,8 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming) tim::mpi::is_initialized_callback() = []() { return false; }; tim::mpi::is_finalized() = true; #else - if(is_root_process()) omnitrace_finalize_hidden(); + if(is_root_process() && omnitrace::get_state() < omnitrace::State::Finalized) + omnitrace_finalize_hidden(); #endif } diff --git a/source/lib/omnitrace/library/components/pthread_create_gotcha.cpp b/source/lib/omnitrace/library/components/pthread_create_gotcha.cpp index 7e1bb88053..aba6097b83 100644 --- a/source/lib/omnitrace/library/components/pthread_create_gotcha.cpp +++ b/source/lib/omnitrace/library/components/pthread_create_gotcha.cpp @@ -102,6 +102,13 @@ inline void stop_bundle(bundle_t& _bundle, int64_t _tid, Args&&... _args) { if(!get_use_timemory() && !get_use_perfetto()) return; + + auto _main_manager = tim::manager::master_instance(); + auto _this_manager = tim::manager::instance(); + if(!_main_manager || !_this_manager || _main_manager->is_finalized() || + _this_manager->is_finalized()) + return; + OMNITRACE_BASIC_VERBOSE_F(3, "stopping bundle '%s' in thread %li...\n", _bundle.key().c_str(), _tid); if(get_use_timemory()) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e17aa21bf9..8d1b9018d1 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -178,7 +178,7 @@ omnitrace_add_test( RUNTIME_ARGS -e -i 256 RUN_ARGS 30 4 1000 ENVIRONMENT - "${_lock_environment};OMNITRACE_USE_TIMEMORY=ON;OMNITRACE_USE_PERFETTO=ON;OMNITRACE_COLLAPSE_THREADS=OFF;OMNITRACE_SAMPLING_REALTIME=ON;OMNITRACE_SAMPLING_REALTIME_FREQ=10;OMNITRACE_SAMPLING_REALTIME_TIDS=0" + "${_lock_environment};OMNITRACE_USE_TIMEMORY=ON;OMNITRACE_USE_PERFETTO=ON;OMNITRACE_COLLAPSE_THREADS=OFF;OMNITRACE_SAMPLING_REALTIME=ON;OMNITRACE_SAMPLING_REALTIME_FREQ=10;OMNITRACE_SAMPLING_REALTIME_TIDS=0;OMNITRACE_SAMPLING_KEEP_INTERNAL=OFF" REWRITE_RUN_PASS_REGEX "wall_clock .*\\|_pthread_create .* 4 .*\\|_pthread_mutex_lock .* 1000 .*\\|_pthread_mutex_unlock .* 1000 .*\\|_pthread_mutex_lock .* 1000 .*\\|_pthread_mutex_unlock .* 1000 .*\\|_pthread_mutex_lock .* 1000 .*\\|_pthread_mutex_unlock .* 1000 .*\\|_pthread_mutex_lock .* 1000 .*\\|_pthread_mutex_unlock .* 1000" RUNTIME_PASS_REGEX @@ -190,10 +190,11 @@ omnitrace_add_test( NAME parallel-overhead-locks-timemory TARGET parallel-overhead-locks LABELS "locks" - REWRITE_ARGS -e -v 2 --min-instructions=4 + REWRITE_ARGS -e -v 2 --min-instructions=32 --dyninst-options InstrStackFrames SaveFPR + TrampRecursive RUN_ARGS 10 4 1000 ENVIRONMENT - "${_lock_environment};OMNITRACE_FLAT_PROFILE=ON;OMNITRACE_USE_TIMEMORY=ON;OMNITRACE_USE_PERFETTO=OFF" + "${_lock_environment};OMNITRACE_FLAT_PROFILE=ON;OMNITRACE_USE_TIMEMORY=ON;OMNITRACE_USE_PERFETTO=OFF;OMNITRACE_SAMPLING_KEEP_INTERNAL=OFF" REWRITE_RUN_PASS_REGEX "start_thread (.*) 4 (.*) pthread_mutex_lock (.*) 4000 (.*) pthread_mutex_unlock (.*) 4000" ) @@ -206,7 +207,7 @@ omnitrace_add_test( REWRITE_ARGS -e -v 2 --min-instructions=8 RUN_ARGS 10 4 1000 ENVIRONMENT - "${_lock_environment};OMNITRACE_FLAT_PROFILE=ON;OMNITRACE_USE_TIMEMORY=OFF;OMNITRACE_USE_PERFETTO=ON" + "${_lock_environment};OMNITRACE_FLAT_PROFILE=ON;OMNITRACE_USE_TIMEMORY=OFF;OMNITRACE_USE_PERFETTO=ON;OMNITRACE_SAMPLING_KEEP_INTERNAL=OFF" ) omnitrace_add_test(