Files
rocm-systems/.github/workflows/linux-ci.yml
T
Jonathan R. Madsen 145a6ae06f omnitrace-dl-library (#25)
* timemory submodule update

* Visibility, setting categories, and task-group protection

- OMNITRACE_VISIBILITY instead of TIMEMORY_VISIBILITY
- increased task group data-race protection
- add omnitrace categories to settings

* set component_apis type-trait

* omnitrace-dl-library implementation

- this library dlopen + dlsym's libomnitrace
- significantly reduces the instrumentation time

* omnitrace-avail categories

- suppress AVAILABLE column when --available

* omnitrace-exe update

- uses omnitrace-dl
- adds --print-excluded option
- removes --jump option
- comments out --stubs option
- removes --stdlib option
- support for C++ STL functions not in libstdc++
- tweak the --print-* outputs
- significantly refactors instrument_module and instrument_entity
- removes unused c_stdlib_module_constraint
- removes unused c_stdlib_function_constraint
- decreases get_whole_function_names() coverage

* library.cpp updates

- OMNITRACE_DEBUG -> OMNITRACE_DEBUG_F
- omnitrace_finalize sets state earlier
- omnitrace_finalize clears push/pop functors
- increased tasking shutdown safety

* - fix critical-trace thread hierarchy
- signal handler calls omnitrace_finalize
- get_cpu_cid_stack supports parent tid
- interval data reserves
- omnitrace-avail serialization support for module_functions
- omnitrace --simulate option
- omnitrace --print-format option
- omnitrace --load-instr option
- omnitrace runtime-inst doesn't oneTimeCode
- updated regex
- expand get_whole_function_names()
- Test Install CI update

* fixes to last commit

- expand get_whole_function_names()
- ignore sig c modules
- kill process in signal handler

* Remove RTLD_DEEPBIND + more

- removed use of RTLD_DEEPBIND
  - causes dyninst segfaults
- fixed signal handling
- updated timemory submodule

* Build/link static timemory libraries

* omnitrace --{module,function}-restrict option

- Added restrict regex options
- Reworked handling of regex options
- Reworked reporting of module/function skipping
- Handle -o w/o file specified

* timemory-avail

- category views
- backtrace::sample checks state

* get_debug_sampling()
2022-02-23 06:59:32 -06:00

553 rivejä
19 KiB
YAML

name: linux-ci
on:
push:
branches: [ main, develop ]
pull_request:
branches: [ main, develop ]
env:
BUILD_TYPE: Release
ELFUTILS_DOWNLOAD_VERSION: 0.183
OMNITRACE_DEBUG_FINALIZE: ON
OMNITRACE_VERBOSE: 1
jobs:
ubuntu-focal:
runs-on: ubuntu-20.04
strategy:
matrix:
compiler: ['g++']
mpi: [ '', 'libmpich-dev mpich', 'libopenmpi-dev openmpi-bin libfabric-dev' ]
steps:
- uses: actions/checkout@v2
- name: Install Packages
timeout-minutes: 5
run:
sudo apt-get update &&
sudo apt-get install -y build-essential m4 autoconf libtool python3-pip libtbb-dev libboost-{atomic,system,thread,date-time,filesystem,timer}-dev ${{ matrix.compiler }} ${{ matrix.mpi }} &&
python3 -m pip install --upgrade pip &&
python3 -m pip install 'cmake==3.16.3'
- name: Configure Env
run:
echo "CC=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')" >> $GITHUB_ENV &&
echo "CXX=${{ matrix.compiler }}" >> $GITHUB_ENV &&
echo "/opt/omnitrace/bin:${HOME}/.local/bin" >> $GITHUB_PATH &&
echo "LD_LIBRARY_PATH=/opt/omnitrace/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
- name: Configure CMake
timeout-minutes: 5
run:
cmake --version &&
if [ -z "${{ matrix.mpi }}" ]; then USE_MPI=OFF; else USE_MPI=ON; fi &&
cmake -B ${{ github.workspace }}/build
-DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')
-DCMAKE_CXX_COMPILER=${{ matrix.compiler }}
-DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }}
-DCMAKE_INSTALL_PREFIX=/opt/omnitrace
-DOMNITRACE_BUILD_TESTING=ON
-DOMNITRACE_BUILD_DYNINST=ON
-DOMNITRACE_USE_MPI=${USE_MPI}
-DOMNITRACE_USE_HIP=OFF
-DDYNINST_BUILD_ELFUTILS=ON
-DDYNINST_BUILD_LIBIBERTY=ON
-DDYNINST_BUILD_SHARED_LIBS=ON
-DDYNINST_BUILD_STATIC_LIBS=OFF
- name: Build
timeout-minutes: 45
run:
cmake --build ${{ github.workspace }}/build --target all --parallel 2 -- VERBOSE=1
- name: Install
run:
cmake --build ${{ github.workspace }}/build --target install --parallel 2
- name: Test
timeout-minutes: 30
working-directory: ${{ github.workspace }}/build
run:
ctest -V --output-log ${{ github.workspace }}/build/omnitrace-ctest-ubuntu-focal.log
- name: Test Install
timeout-minutes: 10
run: |
set -v
export OMNITRACE_DEBUG=ON
which omnitrace-avail
ldd $(which omnitrace-avail)
omnitrace-avail --help
omnitrace-avail -a
which omnitrace-critical-trace
ldd $(which omnitrace-critical-trace)
which omnitrace
ldd $(which omnitrace)
omnitrace --help
omnitrace -e -v 1 -o ls.inst --simulate -- ls
for i in omnitrace-ls.inst-output/*; do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done
omnitrace -e -v 1 -o ls.inst -- ls
./ls.inst
omnitrace -e -v 1 --simulate -- ls
for i in omnitrace-ls-output/*; do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done
omnitrace -e -v 1 -- ls
- name: CTest Artifacts
uses: actions/upload-artifact@v2
with:
name: ctest-log
path: |
${{ github.workspace }}/build/*.log
- name: Data Artifacts
uses: actions/upload-artifact@v2
with:
name: data-files
path: |
${{ github.workspace }}/build/omnitrace-tests-output/*.txt
ubuntu-bionic:
runs-on: ubuntu-18.04
strategy:
matrix:
compiler: ['g++-7', 'g++-8']
mpi: [ '', 'libmpich-dev mpich' ]
steps:
- uses: actions/checkout@v2
- name: Install Packages
timeout-minutes: 5
run:
sudo apt-get update &&
sudo apt-get install -y build-essential m4 autoconf libtool python3-pip ${{ matrix.compiler }} ${{ matrix.mpi }} &&
python3 -m pip install --upgrade pip &&
python3 -m pip install 'cmake==3.16.3'
- name: Configure Env
run:
echo "CC=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')" >> $GITHUB_ENV &&
echo "CXX=${{ matrix.compiler }}" >> $GITHUB_ENV &&
echo "/opt/omnitrace/bin:${HOME}/.local/bin" >> $GITHUB_PATH &&
echo "LD_LIBRARY_PATH=/opt/omnitrace/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
- name: Configure CMake
timeout-minutes: 5
run:
cmake --version &&
if [ -z "${{ matrix.mpi }}" ]; then USE_MPI=OFF; else USE_MPI=ON; fi &&
cmake -B ${{ github.workspace }}/build
-DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')
-DCMAKE_CXX_COMPILER=${{ matrix.compiler }}
-DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }}
-DCMAKE_INSTALL_PREFIX=/opt/omnitrace
-DOMNITRACE_BUILD_TESTING=ON
-DOMNITRACE_BUILD_DYNINST=ON
-DOMNITRACE_USE_MPI=${USE_MPI}
-DOMNITRACE_USE_HIP=OFF
-DDYNINST_BUILD_TBB=ON
-DDYNINST_BUILD_BOOST=ON
-DDYNINST_BUILD_ELFUTILS=ON
-DDYNINST_BUILD_LIBIBERTY=ON
- name: Build
timeout-minutes: 45
run:
cmake --build ${{ github.workspace }}/build --target all --parallel 2 -- VERBOSE=1
- name: Install
run:
cmake --build ${{ github.workspace }}/build --target install --parallel 2
- name: Test
timeout-minutes: 30
working-directory: ${{ github.workspace }}/build
run:
ctest -V --output-log ${{ github.workspace }}/build/omnitrace-ctest-ubuntu-bionic.log
- name: Test Install
timeout-minutes: 10
run: |
set -v
export OMNITRACE_DEBUG=ON
which omnitrace-avail
ldd $(which omnitrace-avail)
omnitrace-avail --help
omnitrace-avail -a
which omnitrace-critical-trace
ldd $(which omnitrace-critical-trace)
which omnitrace
ldd $(which omnitrace)
omnitrace --help
omnitrace -e -v 1 -o ls.inst --simulate -- ls
for i in omnitrace-ls.inst-output/*; do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done
omnitrace -e -v 1 -o ls.inst -- ls
./ls.inst
omnitrace -e -v 1 --simulate -- ls
for i in omnitrace-ls-output/*; do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done
omnitrace -e -v 1 -- ls
- name: CTest Artifacts
uses: actions/upload-artifact@v2
with:
name: ctest-log
path: |
${{ github.workspace }}/build/*.log
- name: Data Artifacts
uses: actions/upload-artifact@v2
with:
name: data-files
path: |
${{ github.workspace }}/build/omnitrace-tests-output/*.txt
ubuntu-focal-external:
runs-on: ubuntu-20.04
strategy:
matrix:
compiler: ['g++-7', 'g++-8', 'g++-9', 'g++-10']
steps:
- uses: actions/checkout@v2
- name: Install Packages
timeout-minutes: 5
run:
sudo apt-get update &&
sudo apt-get install -y build-essential m4 autoconf libtool python3-pip libboost-{atomic,system,thread,date-time,filesystem,timer}-dev libtbb-dev libiberty-dev ${{ matrix.compiler }} &&
sudo python3 -m pip install --upgrade pip &&
python3 -m pip install 'cmake==3.16.3'
- name: Configure Env
run:
echo "CC=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')" >> $GITHUB_ENV &&
echo "CXX=${{ matrix.compiler }}" >> $GITHUB_ENV &&
echo "CMAKE_PREFIX_PATH=/opt/dyninst:/opt/elfutils:${CMAKE_PREFIX_PATH}" >> $GITHUB_ENV &&
echo "/opt/omnitrace/bin:/opt/dyninst/bin:/opt/elfutils/bin:${HOME}/.local/bin" >> $GITHUB_PATH &&
echo "LD_LIBRARY_PATH=/opt/omnitrace/lib:/opt/dyninst/lib:/opt/elfutils/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
- name: Install ElfUtils
timeout-minutes: 5
run:
pushd external &&
wget https://sourceware.org/elfutils/ftp/${ELFUTILS_DOWNLOAD_VERSION}/elfutils-${ELFUTILS_DOWNLOAD_VERSION}.tar.bz2 &&
tar xjf elfutils-${ELFUTILS_DOWNLOAD_VERSION}.tar.bz2 &&
pushd elfutils-${ELFUTILS_DOWNLOAD_VERSION} &&
CFLAGS="-g -O2" ./configure --enable-install-elfh --prefix=/opt/elfutils --disable-libdebuginfod --disable-debuginfod &&
make -j2 &&
make install -j2 &&
popd &&
rm -rf elfutils*
- name: Install Dyninst
timeout-minutes: 25
run:
cmake --version &&
git submodule update --init external/dyninst &&
cd external/dyninst &&
cmake -B build
-DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')
-DCMAKE_CXX_COMPILER=${{ matrix.compiler }}
-DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_PREFIX=/opt/dyninst &&
cmake --build build --target all --parallel 2 &&
cmake --build build --target install --parallel 2 &&
rm -rf build
- name: Configure CMake
timeout-minutes: 5
run:
cmake --version &&
cmake -B ${{ github.workspace }}/build
-DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')
-DCMAKE_CXX_COMPILER=${{ matrix.compiler }}
-DCMAKE_BUILD_TYPE=RelWithDebInfo
-DCMAKE_INSTALL_PREFIX=/opt/omnitrace
-DOMNITRACE_BUILD_TESTING=ON
-DOMNITRACE_USE_MPI=OFF
-DOMNITRACE_USE_HIP=OFF
- name: Build
timeout-minutes: 45
run:
cmake --build ${{ github.workspace }}/build --target all --parallel 2 -- VERBOSE=1
- name: Install
run:
cmake --build ${{ github.workspace }}/build --target install --parallel 2
- name: Test
timeout-minutes: 30
working-directory: ${{ github.workspace }}/build
run:
ldd ./omnitrace &&
./omnitrace --help &&
ctest -V --output-log ${{ github.workspace }}/build/omnitrace-ctest-ubuntu-focal-external.log
- name: Test Install
timeout-minutes: 10
run: |
set -v
export OMNITRACE_DEBUG=ON
which omnitrace-avail
ldd $(which omnitrace-avail)
omnitrace-avail --help
omnitrace-avail -a
which omnitrace-critical-trace
ldd $(which omnitrace-critical-trace)
which omnitrace
ldd $(which omnitrace)
omnitrace --help
omnitrace -e -v 1 -o ls.inst --simulate -- ls
for i in omnitrace-ls.inst-output/*; do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done
omnitrace -e -v 1 -o ls.inst -- ls
./ls.inst
omnitrace -e -v 1 --simulate -- ls
for i in omnitrace-ls-output/*; do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done
omnitrace -e -v 1 -- ls
- name: CTest Artifacts
uses: actions/upload-artifact@v2
with:
name: ctest-log
path: |
${{ github.workspace }}/build/*.log
- name: Data Artifacts
uses: actions/upload-artifact@v2
with:
name: data-files
path: |
${{ github.workspace }}/build/omnitrace-tests-output/*.txt
ubuntu-focal-dyninst-package:
runs-on: ubuntu-20.04
strategy:
matrix:
compiler: ['g++']
steps:
- uses: actions/checkout@v2
- name: Install Packages
timeout-minutes: 5
run:
sudo apt-get update &&
sudo apt-get install -y build-essential m4 autoconf libtool python3-pip ${{ matrix.compiler }} &&
sudo python3 -m pip install --upgrade pip &&
python3 -m pip install 'cmake==3.16.3'
- name: Configure Env
run:
echo "CC=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')" >> $GITHUB_ENV &&
echo "CXX=${{ matrix.compiler }}" >> $GITHUB_ENV &&
echo "CMAKE_PREFIX_PATH=/opt/dyninst:${CMAKE_PREFIX_PATH}" >> $GITHUB_ENV &&
echo "/opt/omnitrace/bin:/opt/dyninst/bin:${HOME}/.local/bin" >> $GITHUB_PATH &&
echo "LD_LIBRARY_PATH=/opt/omnitrace/lib:/opt/dyninst/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
- name: Install Dyninst
timeout-minutes: 25
run:
cmake --version &&
git submodule update --init external/dyninst &&
cd external/dyninst &&
cmake -B build
-DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')
-DCMAKE_CXX_COMPILER=${{ matrix.compiler }}
-DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }}
-DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/external/dyninst-install
-DBUILD_TBB=ON
-DBUILD_BOOST=ON
-DBUILD_ELFUTILS=ON
-DBUILD_LIBIBERTY=ON &&
cmake --build build --target package --parallel 4 &&
mkdir /opt/dyninst &&
./build/Dyninst-*-Linux.sh --prefix=/opt/dyninst --exclude-subdir --skip-license &&
rm -rf build
- name: Configure CMake
timeout-minutes: 5
run:
cmake --version &&
cmake -B ${{ github.workspace }}/build
-DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')
-DCMAKE_CXX_COMPILER=${{ matrix.compiler }}
-DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }}
-DCMAKE_INSTALL_PREFIX=/opt/omnitrace
-DOMNITRACE_BUILD_TESTING=ON
-DOMNITRACE_USE_MPI=OFF
-DOMNITRACE_USE_HIP=OFF
- name: Build
timeout-minutes: 45
run:
cmake --build ${{ github.workspace }}/build --target all --parallel 2 -- VERBOSE=1
- name: Install
run:
cmake --build ${{ github.workspace }}/build --target install --parallel 2
- name: Test
timeout-minutes: 30
working-directory: ${{ github.workspace }}/build
run:
ldd ./omnitrace &&
./omnitrace --help &&
ctest -V --output-log ${{ github.workspace }}/build/omnitrace-ctest-ubuntu-focal-dyninst-package.log
- name: Test Install
timeout-minutes: 10
run: |
set -v
export OMNITRACE_DEBUG=ON
which omnitrace-avail
ldd $(which omnitrace-avail)
omnitrace-avail --help
omnitrace-avail -a
which omnitrace-critical-trace
ldd $(which omnitrace-critical-trace)
which omnitrace
ldd $(which omnitrace)
omnitrace --help
omnitrace -e -v 1 -o ls.inst --simulate -- ls
for i in omnitrace-ls.inst-output/*; do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done
omnitrace -e -v 1 -o ls.inst -- ls
./ls.inst
omnitrace -e -v 1 --simulate -- ls
for i in omnitrace-ls-output/*; do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done
omnitrace -e -v 1 -- ls
- name: CTest Artifacts
uses: actions/upload-artifact@v2
with:
name: ctest-log
path: |
${{ github.workspace }}/build/*.log
- name: Data Artifacts
uses: actions/upload-artifact@v2
with:
name: data-files
path: |
${{ github.workspace }}/build/omnitrace-tests-output/*.txt
ubuntu-focal-external-rocm:
runs-on: ubuntu-20.04
strategy:
matrix:
compiler: ['g++']
rocm_version: ['4.3', '4.3.1', '4.5']
mpi: [ 'libmpich-dev mpich', 'libopenmpi-dev openmpi-bin libfabric-dev' ]
steps:
- uses: actions/checkout@v2
- name: Install Packages
timeout-minutes: 5
run:
echo '1' | sudo tee /proc/sys/kernel/perf_event_paranoid &&
sudo apt-get update &&
sudo apt-get install -y software-properties-common wget gnupg2 &&
sudo wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | sudo apt-key add - &&
echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${{ matrix.rocm_version }}/ ubuntu main" | sudo tee /etc/apt/sources.list.d/rocm.list &&
sudo apt-get update &&
sudo apt-get install -y build-essential m4 autoconf libtool python3-pip libboost-{atomic,system,thread,date-time,filesystem,timer}-dev libtbb-dev libiberty-dev ${{ matrix.compiler }} libudev-dev libnuma-dev rocm-dev rocm-utils roctracer-dev rocprofiler-dev hip-base hsa-amd-aqlprofile hsa-rocr-dev hsakmt-roct-dev ${{ matrix.mpi }} libpapi-dev &&
sudo python3 -m pip install --upgrade pip &&
python3 -m pip install 'cmake==3.16.3'
- name: Configure Env
run:
echo "CC=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')" >> $GITHUB_ENV &&
echo "CXX=${{ matrix.compiler }}" >> $GITHUB_ENV &&
echo "CMAKE_PREFIX_PATH=/opt/dyninst:/opt/elfutils:${CMAKE_PREFIX_PATH}" >> $GITHUB_ENV &&
echo "/opt/omnitrace/bin:/opt/dyninst/bin:/opt/elfutils/bin:${HOME}/.local/bin" >> $GITHUB_PATH &&
echo "LD_LIBRARY_PATH=/opt/omnitrace/lib:/opt/dyninst/lib:/opt/elfutils/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
- name: Install ElfUtils
timeout-minutes: 5
run:
pushd external &&
wget https://sourceware.org/elfutils/ftp/${ELFUTILS_DOWNLOAD_VERSION}/elfutils-${ELFUTILS_DOWNLOAD_VERSION}.tar.bz2 &&
tar xjf elfutils-${ELFUTILS_DOWNLOAD_VERSION}.tar.bz2 &&
pushd elfutils-${ELFUTILS_DOWNLOAD_VERSION} &&
CFLAGS="-O3" ./configure --enable-install-elfh --prefix=/opt/elfutils --disable-libdebuginfod --disable-debuginfod &&
make -j2 &&
make install -j2 &&
popd &&
rm -rf elfutils*
- name: Install Dyninst
timeout-minutes: 25
run:
cmake --version &&
git submodule update --init external/dyninst &&
cd external/dyninst &&
cmake -B build
-DCMAKE_C_COMPILER=${CC}
-DCMAKE_CXX_COMPILER=${CXX}
-DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_PREFIX=/opt/dyninst &&
cmake --build build --target all --parallel 2 &&
cmake --build build --target install --parallel 2 &&
rm -rf build
- name: Configure CMake
timeout-minutes: 5
run:
cmake --version &&
cmake -B ${{ github.workspace }}/build
-DCMAKE_C_COMPILER=${CC}
-DCMAKE_CXX_COMPILER=${CXX}
-DCMAKE_BUILD_TYPE=RelWithDebInfo
-DCMAKE_INSTALL_PREFIX=/opt/omnitrace
-DOMNITRACE_BUILD_TESTING=OFF
-DOMNITRACE_BUILD_DEVELOPER=ON
-DOMNITRACE_BUILD_EXTRA_OPTIMIZATIONS=OFF
-DOMNITRACE_BUILD_LTO=OFF
-DOMNITRACE_USE_MPI=OFF
-DOMNITRACE_USE_MPI_HEADERS=ON
-DOMNITRACE_USE_HIP=ON
-DOMNITRACE_MAX_THREADS=256
-DOMNITRACE_USE_SANITIZER=OFF
-DTIMEMORY_USE_PAPI=ON
- name: Build
timeout-minutes: 45
run:
cmake --build ${{ github.workspace }}/build --target all --parallel 2 -- VERBOSE=1
- name: Install
run:
cmake --build ${{ github.workspace }}/build --target install --parallel 2
- name: Test
timeout-minutes: 30
working-directory: ${{ github.workspace }}/build
run:
ldd ./omnitrace &&
./omnitrace --help
- name: Test Install
timeout-minutes: 10
run: |
set -v
export OMNITRACE_DEBUG=ON
which omnitrace-avail
ldd $(which omnitrace-avail)
omnitrace-avail --help
omnitrace-avail -a
which omnitrace-critical-trace
ldd $(which omnitrace-critical-trace)
which omnitrace
ldd $(which omnitrace)
omnitrace --help
omnitrace -e -v 1 -o ls.inst --simulate -- ls
for i in omnitrace-ls.inst-output/*; do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done
omnitrace -e -v 1 -o ls.inst -- ls
./ls.inst
omnitrace -e -v 1 --simulate -- ls
for i in omnitrace-ls-output/*; do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done
omnitrace -e -v 1 -- ls