User api updates (#32)

* Update invoke.hpp

* Update OMNITRACE_FUNCTION

* Update library debug messages

* ptl verbosity

* Update timemory submodule

* mpi_gotcha calls omnitrace_finalize_hidden

* omnitrace_{push,pop}_region returns error code

* omnitrace-user updates

- doxygen documentation
- omnitrace_get_user_callbacks
- omnitrace_user_error_string
- omnitrace-user functions return error codes

* Update user-api example

* Tweak to workflows and tests

* Fix for OMNITRACE_FUNCTION

- conditional impl if __GNUC__ < 9

* focal-external-rocm workflow update

examples/user-api/user-api.cpp

@@ -2,6 +2,7 @@
 #include <omnitrace/user.h>
 
 #include <atomic>
+#include <cassert>
 #include <cstdio>
 #include <cstdlib>
 #include <sstream>
@@ -16,33 +17,25 @@ fib(long n) __attribute__((noinline));
 void
 run(size_t nitr, long) __attribute__((noinline));
 
-long
-fib(long n)
-{
-    return (n < 2) ? n : fib(n - 1) + fib(n - 2);
-}
+int
+custom_push_region(const char* name);
 
-#define RUN_LABEL                                                                        \
-    std::string{ std::string{ __FUNCTION__ } + "(" + std::to_string(n) + ") x " +        \
-                 std::to_string(nitr) }                                                  \
-        .c_str()
-
-void
-run(size_t nitr, long n)
+namespace
 {
-    omnitrace_user_stop_thread_trace();
-    omnitrace_user_push_region(RUN_LABEL);
-    long local = 0;
-    for(size_t i = 0; i < nitr; ++i)
-        local += fib(n);
-    total += local;
-    omnitrace_user_pop_region(RUN_LABEL);
-    omnitrace_user_start_thread_trace();
+int (*omnitrace_push_region_f)(const char*) = nullptr;
 }
 
 int
 main(int argc, char** argv)
 {
+    // get the internal callback to start a user-defined region
+    omnitrace_user_get_callbacks(OMNITRACE_USER_REGION, (void**) &omnitrace_push_region_f,
+                                 nullptr);
+    // assign the custom callback to start a user-defined region
+    if(omnitrace_push_region_f)
+        omnitrace_user_configure(OMNITRACE_USER_REGION, (void*) &custom_push_region,
+                                 nullptr);
+
     omnitrace_user_push_region(argv[0]);
     omnitrace_user_push_region("initialization");
     size_t nthread = std::min<size_t>(16, std::thread::hardware_concurrency());
@@ -59,22 +52,54 @@ main(int argc, char** argv)
     omnitrace_user_push_region("thread_creation");
     std::vector<std::thread> threads{};
     threads.reserve(nthread);
+    // disable instrumentation for child threads
+    omnitrace_user_stop_thread_trace();
     for(size_t i = 0; i < nthread; ++i)
     {
-        size_t _nitr = ((i % 2) == 1) ? (nitr - (0.1 * nitr)) : (nitr + (0.1 * nitr));
-        threads.emplace_back(&run, _nitr, nfib);
+        threads.emplace_back(&run, nitr, nfib);
     }
+    // re-enable instrumentation
+    omnitrace_user_start_thread_trace();
     omnitrace_user_pop_region("thread_creation");
 
-    run(nitr - 0.25 * nitr, nfib - 0.1 * nfib);
-
     omnitrace_user_push_region("thread_wait");
     for(auto& itr : threads)
         itr.join();
     omnitrace_user_pop_region("thread_wait");
 
+    run(nitr, nfib);
+
     printf("[%s] fibonacci(%li) x %lu = %li\n", argv[0], nfib, nthread, total.load());
     omnitrace_user_pop_region(argv[0]);
 
     return 0;
 }
+
+long
+fib(long n)
+{
+    return (n < 2) ? n : fib(n - 1) + fib(n - 2);
+}
+
+#define RUN_LABEL                                                                        \
+    std::string{ std::string{ __FUNCTION__ } + "(" + std::to_string(n) + ") x " +        \
+                 std::to_string(nitr) }                                                  \
+        .c_str()
+
+void
+run(size_t nitr, long n)
+{
+    omnitrace_user_push_region(RUN_LABEL);
+    long local = 0;
+    for(size_t i = 0; i < nitr; ++i)
+        local += fib(n);
+    total += local;
+    omnitrace_user_pop_region(RUN_LABEL);
+}
+
+int
+custom_push_region(const char* name)
+{
+    printf("Pushing custom region :: %s\n", name);
+    return (*omnitrace_push_region_f)(name);
+}