restructure libomnitrace + tasking and omnitrace-causal updates (#237)

* restructured libomnitrace

- this is necessary to incorporate some of the binary analysis capabilities into omnitrace exe
- created libomnitrace-core (static)
- created libomnitrace-binary (static)
- created libomnitrace (static)
- omnitrace-avail links to libomnitrace.a
- omnitrace-critical-trace links to libomnitrace.a
- tweaked the testing
  - reduced verbosity on some of MPI tests
  - excluded trace-time-window from tests on Ubuntu 18.04
  - reduced causal e2e iterations
- minor tweak to tasking
  - manually create `PTL::UserTaskQueue` instance instead of relying on `PTL::ThreadPool` to create it

* Update formatting workflow

- source formatting uses ubuntu-22.04
- check-includes doesn't generate false positive for 'include "timemory.hpp"'

* omnitrace-causal --generate-configs

- fix config generation in omnitrace causal
- add test for omnitrace-causal + generating configs

* Fix omnitrace-object-library build

- accidentally included rocm sources in non-rocm builds

* Fix rocm compilation w/o rocprofiler

* update timemory submodule with mpi_get warning messages

* sampling offload file updates

- more verbose messages
- disable offload before stopping

* testing updates

- increase causal e2e iterations to 12
- increase lock_environment verbose to 2 (for sampling offload messages)
- fix return for omnitrace_add_validation_test

source/bin/omnitrace-avail/CMakeLists.txt

@@ -20,13 +20,14 @@ target_sources(
             ${CMAKE_CURRENT_LIST_DIR}/get_availability.hpp
             ${CMAKE_CURRENT_LIST_DIR}/get_categories.hpp
             ${CMAKE_CURRENT_LIST_DIR}/info_type.cpp
-            ${CMAKE_CURRENT_LIST_DIR}/info_type.hpp
-            $<TARGET_OBJECTS:omnitrace::omnitrace-object-library>)
+            ${CMAKE_CURRENT_LIST_DIR}/info_type.hpp)
 
 target_include_directories(omnitrace-avail PRIVATE ${CMAKE_CURRENT_LIST_DIR})
 target_compile_definitions(omnitrace-avail PRIVATE OMNITRACE_EXTERN_COMPONENTS=0)
-target_link_libraries(omnitrace-avail PRIVATE omnitrace::omnitrace-compile-definitions
-                                              omnitrace::omnitrace-interface-library)
+target_link_libraries(
+    omnitrace-avail
+    PRIVATE omnitrace::omnitrace-compile-definitions
+            omnitrace::omnitrace-interface-library omnitrace::libomnitrace-static)
 set_target_properties(
     omnitrace-avail PROPERTIES BUILD_RPATH "\$ORIGIN:\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"
                                INSTALL_RPATH "${OMNITRACE_EXE_INSTALL_RPATH}")

source/bin/omnitrace-avail/avail.cpp

@@ -29,8 +29,8 @@
 #include "get_availability.hpp"
 #include "info_type.hpp"
 
-#include "library/config.hpp"
-#include "library/gpu.hpp"
+#include "core/config.hpp"
+#include "core/gpu.hpp"
 #include "library/rocprofiler.hpp"
 
 #include <timemory/components.hpp>

source/bin/omnitrace-avail/defines.hpp

@@ -25,5 +25,6 @@
 #define TIMEMORY_DISABLE_BANNER
 #define TIMEMORY_DISABLE_COMPONENT_STORAGE_INIT
 
-#include "library/config.hpp"
-#include "library/defines.hpp"
+#include "common/defines.h"
+#include "core/config.hpp"
+#include "core/defines.hpp"

source/bin/omnitrace-causal/impl.cpp

@@ -905,11 +905,12 @@ parse_args(int argc, char** argv, std::vector<char*>& _env,
     {
         auto _is_omni_cfg = [](std::string_view itr) {
             return (itr.find("OMNITRACE") == 0 && itr.find("OMNITRACE_MODE") != 0 &&
-                    itr.find("OMNITRACE_CONFIG_FILE") != 0 &&
-                    itr.find('=') < itr.length());
+                    itr.find("OMNITRACE_DEBUG_") != 0 && itr.find('=') < itr.length());
+            // omnitrace has miscellaneous env options starting with OMNITRACE_DEBUG_ that
+            // are not official options
         };
 
-        auto _omni_env = std::map<std::string, std::string>{};
+        auto _omni_env_m = std::map<std::string, std::string>{};
         for(auto* itr : _env)
         {
             if(_is_omni_cfg(itr))
@@ -918,12 +919,25 @@ parse_args(int argc, char** argv, std::vector<char*>& _env,
                 auto _pos     = _env_var.find('=');
                 auto _env_val = _env_var.substr(_pos + 1);
                 _env_var      = _env_var.substr(0, _pos);
-                _omni_env.emplace(_env_var, _env_val);
+                _omni_env_m.emplace(_env_var, _env_val);
             }
         }
 
         _env.erase(std::remove_if(_env.begin(), _env.end(), _is_omni_cfg), _env.end());
 
+        auto _omni_env = std::vector<std::pair<std::string, std::string>>{};
+        // make sure that OMNITRACE_CONFIG_FILE is the first entry
+        {
+            auto citr = _omni_env_m.find("OMNITRACE_CONFIG_FILE");
+            if(citr != _omni_env_m.end())
+            {
+                _omni_env.emplace_back(citr->first, citr->second);
+                _omni_env_m.erase(citr);
+            }
+        }
+        for(const auto& itr : _omni_env_m)
+            _omni_env.emplace_back(itr.first, itr.second);
+
         _causal_envs_tmp = std::move(_causal_envs);
         _causal_envs.clear();
         auto _write_config =

source/bin/omnitrace-critical-trace/CMakeLists.txt

@@ -4,11 +4,8 @@
 #
 # ------------------------------------------------------------------------------#
 
-add_executable(
-    omnitrace-critical-trace
-    ${CMAKE_CURRENT_LIST_DIR}/critical-trace.cpp
-    ${CMAKE_CURRENT_LIST_DIR}/critical-trace.hpp
-    $<TARGET_OBJECTS:omnitrace::omnitrace-object-library>)
+add_executable(omnitrace-critical-trace ${CMAKE_CURRENT_LIST_DIR}/critical-trace.cpp
+                                        ${CMAKE_CURRENT_LIST_DIR}/critical-trace.hpp)
 
 target_include_directories(omnitrace-critical-trace PRIVATE ${CMAKE_CURRENT_LIST_DIR})
 target_compile_definitions(omnitrace-critical-trace PRIVATE OMNITRACE_EXTERN_COMPONENTS=0)
@@ -16,7 +13,7 @@ target_link_libraries(
     omnitrace-critical-trace
     PRIVATE omnitrace::omnitrace-compile-definitions
             omnitrace::omnitrace-interface-library omnitrace::omnitrace-headers
-            omnitrace::omnitrace-timemory)
+            omnitrace::omnitrace-timemory omnitrace::libomnitrace-static)
 set_target_properties(
     omnitrace-critical-trace
     PROPERTIES BUILD_RPATH "\$ORIGIN:\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}"

source/bin/omnitrace-critical-trace/critical-trace.cpp

@@ -23,9 +23,9 @@
 #include "critical-trace.hpp"
 
 #include "api.hpp"
-#include "library/config.hpp"
-#include "library/debug.hpp"
-#include "library/perfetto.hpp"
+#include "core/config.hpp"
+#include "core/debug.hpp"
+#include "core/perfetto.hpp"
 #include "library/tracing.hpp"
 
 #include <timemory/hash/types.hpp>

source/bin/omnitrace-critical-trace/critical-trace.hpp

@@ -22,11 +22,11 @@
 
 #pragma once
 
-#include "library/config.hpp"
+#include "core/config.hpp"
+#include "core/debug.hpp"
+#include "core/defines.hpp"
+#include "core/perfetto.hpp"
 #include "library/critical_trace.hpp"
-#include "library/debug.hpp"
-#include "library/defines.hpp"
-#include "library/perfetto.hpp"
 #include "library/ptl.hpp"
 
 #include <PTL/ThreadPool.hh>