2
0
Ficheiros
rocm-systems/projects/rocprofiler-systems/.github/workflows/linux-ci.yml
T
Jonathan R. Madsen 083035dd8b User API + reorganized lib folders (#30)
* User API + reorganized lib folders

- omnitrace_user_start_trace
- omnitrace_user_stop_trace
- omnitrace_user_start_thread_trace
- omnitrace_user_stop_thread_trace
- omnitrace_user_push_region
- omnitrace_user_pop_region

* New OpenMP examples/tests

* Fix to KokkosP

* OMPT support

- fixed omnitrace instrumenting reporting
- common invoke improvements
- component::user_region

* exclude kmp_threadprivate_

* Separate omnitrace into multiple files

* PTL and timemory submodule updates

* Active guards + USE_OMPT guards in omnitrace-dl

* Tweak transpose default iterations

* omnitrace-precommit build target

* Omnitrace exe restructuring pt 2

- Never instrument functions with less than 4 instructions
- Never instrument ompt_start_tool or nanosleep
- module_function serializes heuristics
- removed hash stuff from omnitrace
- removed instr_procedures lambda
- WAITPID_DEBUG_MESSAGE

* set_state, "_hidden" fix, CI exceptions, backtrace fix

- set_state function
- fixed "_hidden" from appearing in print macros using __FUNCTION__
- OMNITRACE_CI_THROW
- more CI checks in library
- fixed backtrace init value sample issue being ignored

* Tweaks to OMPT tests

* cmake-formatting

* Removed debug output from backtrace processing

* Fix warnings and verbosity

* omnitrace-dl fix for libomp

* omnitrace-avail fixes

- remove second omnitrace_init_library call
- fix -r option not working

* Additional testing

- source/bin/tests
- tests for omnitrace-exe
- tests for omnitrace-avail

* cmake-format

* Reduce runtime of openmp-lu

* Update openmp-lu and tests timeout

* openmp-lu and CI tweaks

- decrease iterations
- OMP_NUM_THREADS=2
- install clang and libomp-dev in linux-ci
- fix data-files in linux-ci

[ROCm/rocprofiler-systems commit: d80752bc69]
2022-03-07 20:40:48 -06:00

554 linhas
19 KiB
YAML

name: linux-ci
on:
push:
branches: [ main, develop ]
pull_request:
branches: [ main, develop ]
env:
BUILD_TYPE: Release
ELFUTILS_DOWNLOAD_VERSION: 0.183
OMNITRACE_DEBUG_FINALIZE: ON
OMNITRACE_VERBOSE: 1
OMNITRACE_CI: ON
jobs:
ubuntu-focal:
runs-on: ubuntu-20.04
strategy:
matrix:
compiler: ['g++']
mpi: [ '', 'libmpich-dev mpich', 'libopenmpi-dev openmpi-bin libfabric-dev' ]
steps:
- uses: actions/checkout@v2
- name: Install Packages
timeout-minutes: 5
run:
sudo apt-get update &&
sudo apt-get install -y build-essential m4 autoconf libtool python3-pip libtbb-dev libboost-{atomic,system,thread,date-time,filesystem,timer}-dev clang libomp-dev ${{ matrix.compiler }} ${{ matrix.mpi }} &&
python3 -m pip install --upgrade pip &&
python3 -m pip install 'cmake==3.16.3'
- name: Configure Env
run:
echo "CC=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')" >> $GITHUB_ENV &&
echo "CXX=${{ matrix.compiler }}" >> $GITHUB_ENV &&
echo "/opt/omnitrace/bin:${HOME}/.local/bin" >> $GITHUB_PATH &&
echo "LD_LIBRARY_PATH=/opt/omnitrace/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
- name: Configure CMake
timeout-minutes: 5
run:
cmake --version &&
if [ -z "${{ matrix.mpi }}" ]; then USE_MPI=OFF; else USE_MPI=ON; fi &&
cmake -B ${{ github.workspace }}/build
-DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')
-DCMAKE_CXX_COMPILER=${{ matrix.compiler }}
-DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }}
-DCMAKE_INSTALL_PREFIX=/opt/omnitrace
-DOMNITRACE_BUILD_TESTING=ON
-DOMNITRACE_BUILD_DYNINST=ON
-DOMNITRACE_USE_MPI=${USE_MPI}
-DOMNITRACE_USE_HIP=OFF
-DDYNINST_BUILD_ELFUTILS=ON
-DDYNINST_BUILD_LIBIBERTY=ON
-DDYNINST_BUILD_SHARED_LIBS=ON
-DDYNINST_BUILD_STATIC_LIBS=OFF
- name: Build
timeout-minutes: 45
run:
cmake --build ${{ github.workspace }}/build --target all --parallel 2 -- VERBOSE=1
- name: Install
run:
cmake --build ${{ github.workspace }}/build --target install --parallel 2
- name: Test
timeout-minutes: 30
working-directory: ${{ github.workspace }}/build
run:
ctest -V --output-log ${{ github.workspace }}/build/omnitrace-ctest-ubuntu-focal.log
- name: Test Install
timeout-minutes: 10
run: |
set -v
export OMNITRACE_DEBUG=ON
which omnitrace-avail
ldd $(which omnitrace-avail)
omnitrace-avail --help
omnitrace-avail -a
which omnitrace-critical-trace
ldd $(which omnitrace-critical-trace)
which omnitrace
ldd $(which omnitrace)
omnitrace --help
omnitrace -e -v 1 -o ls.inst --simulate -- ls
for i in omnitrace-ls.inst-output/*; do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done
omnitrace -e -v 1 -o ls.inst -- ls
./ls.inst
omnitrace -e -v 1 --simulate -- ls
for i in omnitrace-ls-output/*; do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done
omnitrace -e -v 1 -- ls
- name: CTest Artifacts
uses: actions/upload-artifact@v2
with:
name: ctest-log
path: |
${{ github.workspace }}/build/*.log
- name: Data Artifacts
uses: actions/upload-artifact@v2
with:
name: data-files
path: |
${{ github.workspace }}/build/omnitrace-tests-output/**/*.txt
ubuntu-bionic:
runs-on: ubuntu-18.04
strategy:
matrix:
compiler: ['g++-7', 'g++-8']
mpi: [ '', 'libmpich-dev mpich' ]
steps:
- uses: actions/checkout@v2
- name: Install Packages
timeout-minutes: 5
run:
sudo apt-get update &&
sudo apt-get install -y build-essential m4 autoconf libtool python3-pip ${{ matrix.compiler }} ${{ matrix.mpi }} &&
python3 -m pip install --upgrade pip &&
python3 -m pip install 'cmake==3.16.3'
- name: Configure Env
run:
echo "CC=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')" >> $GITHUB_ENV &&
echo "CXX=${{ matrix.compiler }}" >> $GITHUB_ENV &&
echo "/opt/omnitrace/bin:${HOME}/.local/bin" >> $GITHUB_PATH &&
echo "LD_LIBRARY_PATH=/opt/omnitrace/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
- name: Configure CMake
timeout-minutes: 5
run:
cmake --version &&
if [ -z "${{ matrix.mpi }}" ]; then USE_MPI=OFF; else USE_MPI=ON; fi &&
cmake -B ${{ github.workspace }}/build
-DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')
-DCMAKE_CXX_COMPILER=${{ matrix.compiler }}
-DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }}
-DCMAKE_INSTALL_PREFIX=/opt/omnitrace
-DOMNITRACE_BUILD_TESTING=ON
-DOMNITRACE_BUILD_DYNINST=ON
-DOMNITRACE_USE_MPI=${USE_MPI}
-DOMNITRACE_USE_HIP=OFF
-DDYNINST_BUILD_TBB=ON
-DDYNINST_BUILD_BOOST=ON
-DDYNINST_BUILD_ELFUTILS=ON
-DDYNINST_BUILD_LIBIBERTY=ON
- name: Build
timeout-minutes: 45
run:
cmake --build ${{ github.workspace }}/build --target all --parallel 2 -- VERBOSE=1
- name: Install
run:
cmake --build ${{ github.workspace }}/build --target install --parallel 2
- name: Test
timeout-minutes: 30
working-directory: ${{ github.workspace }}/build
run:
ctest -V --output-log ${{ github.workspace }}/build/omnitrace-ctest-ubuntu-bionic.log
- name: Test Install
timeout-minutes: 10
run: |
set -v
export OMNITRACE_DEBUG=ON
which omnitrace-avail
ldd $(which omnitrace-avail)
omnitrace-avail --help
omnitrace-avail -a
which omnitrace-critical-trace
ldd $(which omnitrace-critical-trace)
which omnitrace
ldd $(which omnitrace)
omnitrace --help
omnitrace -e -v 1 -o ls.inst --simulate -- ls
for i in omnitrace-ls.inst-output/*; do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done
omnitrace -e -v 1 -o ls.inst -- ls
./ls.inst
omnitrace -e -v 1 --simulate -- ls
for i in omnitrace-ls-output/*; do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done
omnitrace -e -v 1 -- ls
- name: CTest Artifacts
uses: actions/upload-artifact@v2
with:
name: ctest-log
path: |
${{ github.workspace }}/build/*.log
- name: Data Artifacts
uses: actions/upload-artifact@v2
with:
name: data-files
path: |
${{ github.workspace }}/build/omnitrace-tests-output/**/*.txt
ubuntu-focal-external:
runs-on: ubuntu-20.04
strategy:
matrix:
compiler: ['g++-7', 'g++-8', 'g++-9', 'g++-10']
steps:
- uses: actions/checkout@v2
- name: Install Packages
timeout-minutes: 5
run:
sudo apt-get update &&
sudo apt-get install -y build-essential m4 autoconf libtool python3-pip libboost-{atomic,system,thread,date-time,filesystem,timer}-dev libtbb-dev libiberty-dev clang libomp-dev ${{ matrix.compiler }} &&
sudo python3 -m pip install --upgrade pip &&
python3 -m pip install 'cmake==3.16.3'
- name: Configure Env
run:
echo "CC=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')" >> $GITHUB_ENV &&
echo "CXX=${{ matrix.compiler }}" >> $GITHUB_ENV &&
echo "CMAKE_PREFIX_PATH=/opt/dyninst:/opt/elfutils:${CMAKE_PREFIX_PATH}" >> $GITHUB_ENV &&
echo "/opt/omnitrace/bin:/opt/dyninst/bin:/opt/elfutils/bin:${HOME}/.local/bin" >> $GITHUB_PATH &&
echo "LD_LIBRARY_PATH=/opt/omnitrace/lib:/opt/dyninst/lib:/opt/elfutils/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
- name: Install ElfUtils
timeout-minutes: 5
run:
pushd external &&
wget https://sourceware.org/elfutils/ftp/${ELFUTILS_DOWNLOAD_VERSION}/elfutils-${ELFUTILS_DOWNLOAD_VERSION}.tar.bz2 &&
tar xjf elfutils-${ELFUTILS_DOWNLOAD_VERSION}.tar.bz2 &&
pushd elfutils-${ELFUTILS_DOWNLOAD_VERSION} &&
CFLAGS="-g -O2" ./configure --enable-install-elfh --prefix=/opt/elfutils --disable-libdebuginfod --disable-debuginfod &&
make -j2 &&
make install -j2 &&
popd &&
rm -rf elfutils*
- name: Install Dyninst
timeout-minutes: 25
run:
cmake --version &&
git submodule update --init external/dyninst &&
cd external/dyninst &&
cmake -B build
-DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')
-DCMAKE_CXX_COMPILER=${{ matrix.compiler }}
-DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_PREFIX=/opt/dyninst &&
cmake --build build --target all --parallel 2 &&
cmake --build build --target install --parallel 2 &&
rm -rf build
- name: Configure CMake
timeout-minutes: 5
run:
cmake --version &&
cmake -B ${{ github.workspace }}/build
-DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')
-DCMAKE_CXX_COMPILER=${{ matrix.compiler }}
-DCMAKE_BUILD_TYPE=RelWithDebInfo
-DCMAKE_INSTALL_PREFIX=/opt/omnitrace
-DOMNITRACE_BUILD_TESTING=ON
-DOMNITRACE_USE_MPI=OFF
-DOMNITRACE_USE_HIP=OFF
- name: Build
timeout-minutes: 45
run:
cmake --build ${{ github.workspace }}/build --target all --parallel 2 -- VERBOSE=1
- name: Install
run:
cmake --build ${{ github.workspace }}/build --target install --parallel 2
- name: Test
timeout-minutes: 30
working-directory: ${{ github.workspace }}/build
run:
ldd ./omnitrace &&
./omnitrace --help &&
ctest -V --output-log ${{ github.workspace }}/build/omnitrace-ctest-ubuntu-focal-external.log
- name: Test Install
timeout-minutes: 10
run: |
set -v
export OMNITRACE_DEBUG=ON
which omnitrace-avail
ldd $(which omnitrace-avail)
omnitrace-avail --help
omnitrace-avail -a
which omnitrace-critical-trace
ldd $(which omnitrace-critical-trace)
which omnitrace
ldd $(which omnitrace)
omnitrace --help
omnitrace -e -v 1 -o ls.inst --simulate -- ls
for i in omnitrace-ls.inst-output/*; do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done
omnitrace -e -v 1 -o ls.inst -- ls
./ls.inst
omnitrace -e -v 1 --simulate -- ls
for i in omnitrace-ls-output/*; do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done
omnitrace -e -v 1 -- ls
- name: CTest Artifacts
uses: actions/upload-artifact@v2
with:
name: ctest-log
path: |
${{ github.workspace }}/build/*.log
- name: Data Artifacts
uses: actions/upload-artifact@v2
with:
name: data-files
path: |
${{ github.workspace }}/build/omnitrace-tests-output/**/*.txt
ubuntu-focal-dyninst-package:
runs-on: ubuntu-20.04
strategy:
matrix:
compiler: ['g++']
steps:
- uses: actions/checkout@v2
- name: Install Packages
timeout-minutes: 5
run:
sudo apt-get update &&
sudo apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev ${{ matrix.compiler }} &&
sudo python3 -m pip install --upgrade pip &&
python3 -m pip install 'cmake==3.16.3'
- name: Configure Env
run:
echo "CC=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')" >> $GITHUB_ENV &&
echo "CXX=${{ matrix.compiler }}" >> $GITHUB_ENV &&
echo "CMAKE_PREFIX_PATH=/opt/dyninst:${CMAKE_PREFIX_PATH}" >> $GITHUB_ENV &&
echo "/opt/omnitrace/bin:/opt/dyninst/bin:${HOME}/.local/bin" >> $GITHUB_PATH &&
echo "LD_LIBRARY_PATH=/opt/omnitrace/lib:/opt/dyninst/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
- name: Install Dyninst
timeout-minutes: 25
run:
cmake --version &&
git submodule update --init external/dyninst &&
cd external/dyninst &&
cmake -B build
-DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')
-DCMAKE_CXX_COMPILER=${{ matrix.compiler }}
-DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }}
-DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/external/dyninst-install
-DBUILD_TBB=ON
-DBUILD_BOOST=ON
-DBUILD_ELFUTILS=ON
-DBUILD_LIBIBERTY=ON &&
cmake --build build --target package --parallel 4 &&
mkdir /opt/dyninst &&
./build/Dyninst-*-Linux.sh --prefix=/opt/dyninst --exclude-subdir --skip-license &&
rm -rf build
- name: Configure CMake
timeout-minutes: 5
run:
cmake --version &&
cmake -B ${{ github.workspace }}/build
-DCMAKE_C_COMPILER=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')
-DCMAKE_CXX_COMPILER=${{ matrix.compiler }}
-DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }}
-DCMAKE_INSTALL_PREFIX=/opt/omnitrace
-DOMNITRACE_BUILD_TESTING=ON
-DOMNITRACE_USE_MPI=OFF
-DOMNITRACE_USE_HIP=OFF
- name: Build
timeout-minutes: 45
run:
cmake --build ${{ github.workspace }}/build --target all --parallel 2 -- VERBOSE=1
- name: Install
run:
cmake --build ${{ github.workspace }}/build --target install --parallel 2
- name: Test
timeout-minutes: 30
working-directory: ${{ github.workspace }}/build
run:
ldd ./omnitrace &&
./omnitrace --help &&
ctest -V --output-log ${{ github.workspace }}/build/omnitrace-ctest-ubuntu-focal-dyninst-package.log
- name: Test Install
timeout-minutes: 10
run: |
set -v
export OMNITRACE_DEBUG=ON
which omnitrace-avail
ldd $(which omnitrace-avail)
omnitrace-avail --help
omnitrace-avail -a
which omnitrace-critical-trace
ldd $(which omnitrace-critical-trace)
which omnitrace
ldd $(which omnitrace)
omnitrace --help
omnitrace -e -v 1 -o ls.inst --simulate -- ls
for i in omnitrace-ls.inst-output/*; do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done
omnitrace -e -v 1 -o ls.inst -- ls
./ls.inst
omnitrace -e -v 1 --simulate -- ls
for i in omnitrace-ls-output/*; do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done
omnitrace -e -v 1 -- ls
- name: CTest Artifacts
uses: actions/upload-artifact@v2
with:
name: ctest-log
path: |
${{ github.workspace }}/build/*.log
- name: Data Artifacts
uses: actions/upload-artifact@v2
with:
name: data-files
path: |
${{ github.workspace }}/build/omnitrace-tests-output/**/*.txt
ubuntu-focal-external-rocm:
runs-on: ubuntu-20.04
strategy:
matrix:
compiler: ['g++']
rocm_version: ['4.3', '4.3.1', '4.5']
mpi: [ 'libmpich-dev mpich', 'libopenmpi-dev openmpi-bin libfabric-dev' ]
steps:
- uses: actions/checkout@v2
- name: Install Packages
timeout-minutes: 5
run:
echo '1' | sudo tee /proc/sys/kernel/perf_event_paranoid &&
sudo apt-get update &&
sudo apt-get install -y software-properties-common wget gnupg2 &&
sudo wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | sudo apt-key add - &&
echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${{ matrix.rocm_version }}/ ubuntu main" | sudo tee /etc/apt/sources.list.d/rocm.list &&
sudo apt-get update &&
sudo apt-get install -y build-essential m4 autoconf libtool python3-pip libboost-{atomic,system,thread,date-time,filesystem,timer}-dev libtbb-dev libiberty-dev clang libomp-dev ${{ matrix.compiler }} libudev-dev libnuma-dev rocm-dev rocm-utils roctracer-dev rocprofiler-dev hip-base hsa-amd-aqlprofile hsa-rocr-dev hsakmt-roct-dev ${{ matrix.mpi }} libpapi-dev &&
sudo python3 -m pip install --upgrade pip &&
python3 -m pip install 'cmake==3.16.3'
- name: Configure Env
run:
echo "CC=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')" >> $GITHUB_ENV &&
echo "CXX=${{ matrix.compiler }}" >> $GITHUB_ENV &&
echo "CMAKE_PREFIX_PATH=/opt/dyninst:/opt/elfutils:${CMAKE_PREFIX_PATH}" >> $GITHUB_ENV &&
echo "/opt/omnitrace/bin:/opt/dyninst/bin:/opt/elfutils/bin:${HOME}/.local/bin" >> $GITHUB_PATH &&
echo "LD_LIBRARY_PATH=/opt/omnitrace/lib:/opt/dyninst/lib:/opt/elfutils/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
- name: Install ElfUtils
timeout-minutes: 5
run:
pushd external &&
wget https://sourceware.org/elfutils/ftp/${ELFUTILS_DOWNLOAD_VERSION}/elfutils-${ELFUTILS_DOWNLOAD_VERSION}.tar.bz2 &&
tar xjf elfutils-${ELFUTILS_DOWNLOAD_VERSION}.tar.bz2 &&
pushd elfutils-${ELFUTILS_DOWNLOAD_VERSION} &&
CFLAGS="-O3" ./configure --enable-install-elfh --prefix=/opt/elfutils --disable-libdebuginfod --disable-debuginfod &&
make -j2 &&
make install -j2 &&
popd &&
rm -rf elfutils*
- name: Install Dyninst
timeout-minutes: 25
run:
cmake --version &&
git submodule update --init external/dyninst &&
cd external/dyninst &&
cmake -B build
-DCMAKE_C_COMPILER=${CC}
-DCMAKE_CXX_COMPILER=${CXX}
-DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_PREFIX=/opt/dyninst &&
cmake --build build --target all --parallel 2 &&
cmake --build build --target install --parallel 2 &&
rm -rf build
- name: Configure CMake
timeout-minutes: 5
run:
cmake --version &&
cmake -B ${{ github.workspace }}/build
-DCMAKE_C_COMPILER=${CC}
-DCMAKE_CXX_COMPILER=${CXX}
-DCMAKE_BUILD_TYPE=RelWithDebInfo
-DCMAKE_INSTALL_PREFIX=/opt/omnitrace
-DOMNITRACE_BUILD_TESTING=OFF
-DOMNITRACE_BUILD_DEVELOPER=ON
-DOMNITRACE_BUILD_EXTRA_OPTIMIZATIONS=OFF
-DOMNITRACE_BUILD_LTO=OFF
-DOMNITRACE_USE_MPI=OFF
-DOMNITRACE_USE_MPI_HEADERS=ON
-DOMNITRACE_USE_HIP=ON
-DOMNITRACE_MAX_THREADS=256
-DOMNITRACE_USE_SANITIZER=OFF
-DTIMEMORY_USE_PAPI=ON
- name: Build
timeout-minutes: 45
run:
cmake --build ${{ github.workspace }}/build --target all --parallel 2 -- VERBOSE=1
- name: Install
run:
cmake --build ${{ github.workspace }}/build --target install --parallel 2
- name: Test
timeout-minutes: 30
working-directory: ${{ github.workspace }}/build
run:
ldd ./omnitrace &&
./omnitrace --help
- name: Test Install
timeout-minutes: 10
run: |
set -v
export OMNITRACE_DEBUG=ON
which omnitrace-avail
ldd $(which omnitrace-avail)
omnitrace-avail --help
omnitrace-avail -a
which omnitrace-critical-trace
ldd $(which omnitrace-critical-trace)
which omnitrace
ldd $(which omnitrace)
omnitrace --help
omnitrace -e -v 1 -o ls.inst --simulate -- ls
for i in omnitrace-ls.inst-output/*; do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done
omnitrace -e -v 1 -o ls.inst -- ls
./ls.inst
omnitrace -e -v 1 --simulate -- ls
for i in omnitrace-ls-output/*; do echo -e "\n\n --> ${i} \n\n"; cat ${i}; done
omnitrace -e -v 1 -- ls