Integration Testing (#211)

* Add external/cereal submodule

- used for integration testing

* Update lib/common/container/small_vector.hpp

- documentation notes

* Update tests/apps

- update transpose app (fix build)
- add reproducible-runtime app

* Update include/rocprofiler/fwd.h

- rocprofiler_service_callback_phase_t -> rocprofiler_callback_phase_t

* Update PTL submodule

- fix for task group: submitting tasks from different thread

* Update lib/rocprofiler/hsa/queue.cpp

- CHECK_NOTNULL(_buffer)

* Update lib/rocprofiler/hsa/hsa.cpp

- use buffer::get_buffer instead of manually looking for buffer

* Update lib/rocprofiler/internal_threading.cpp

- use buffer::get_buffer instead of manually looking for buffer

* Update lib/rocprofiler/buffer.cpp

- offset the buffer id
- properly handle rocprofiler_create_buffer reusing rocprofiler_buffer_id_t on a different context

* Update tests

- kernel tracing library for integration testing

* Add cereal submodule

* Update lib/rocprofiler/registration.*

- OnUnload
- Support ROCP_TOOL_LIBRARIES for python usage
- improve finalize function
- remove calling hsa_shut_down in finalize function

* Update lib/rocprofiler/buffer.*

- allocate_buffer sets the buffer id value
- expose (internally) is_valid_buffer_id
- update test

* Update tests/kernel-tracing

- installation
- better organization of JSON groups
- improved messaging

* Update lib/rocprofiler/registration.cpp

- add workaround for hsa-runtime supporting rocprofiler-register

* Update tests/kernel-tracing/kernel-tracing.cpp

- fix memory leaks

* cereal support for minimal JSON

- update cereal submodule to rocprofiler branch
- change REPO_BRANCH in rocprofiler_checkout_git_submodule for cereal
- update tests/kernel-tracing/kernel-tracing.cpp
  - use minimal json
  - slight tweak putting giving contexts name in storing name + context pointer pair in map

* Update tests/kernel-tracing/kernel-tracing.cpp

- support runtime selection of contexts via KERNEL_TRACING_CONTEXTS environment variable

* Update tests

- tests/CMakeLists.txt
  - find_package(Python3 REQUIRED)
- tests/kernel-tracing
  - pytest validation

* Update CI workflow

- install pytest
- add checks for test labels

* Update scripts/run-ci.py

- change --coverage options
  - replace 'unittests' with 'tests'
- replace test label regex '-L unittests' with '-L tests'

* Update requirements.txt

- this is now an empty file since none of the packages are required for this repo

source/include/rocprofiler/fwd.h

@@ -113,7 +113,7 @@ typedef enum  // NOLINT(performance-enum-size)
     ROCPROFILER_CALLBACK_PHASE_UNLOAD =
         ROCPROFILER_CALLBACK_PHASE_EXIT,  ///< Callback invoked prior to code object unloading
     ROCPROFILER_CALLBACK_PHASE_LAST,
-} rocprofiler_service_callback_phase_t;
+} rocprofiler_callback_phase_t;
 
 /**
  * @brief Service Callback Tracing Kind.
@@ -373,7 +373,7 @@ typedef struct rocprofiler_callback_tracing_record_t
     rocprofiler_correlation_id_t                correlation_id;
     rocprofiler_service_callback_tracing_kind_t kind;
     uint32_t                                    operation;
-    rocprofiler_service_callback_phase_t        phase;
+    rocprofiler_callback_phase_t                phase;
     void*                                       payload;
 } rocprofiler_callback_tracing_record_t;
 

source/lib/common/container/small_vector.hpp

@@ -22,6 +22,21 @@
 
 #pragma once
 
+/**
+ * @file small_vector.hpp
+ * @brief This is inspired and largely derived from llvm/ADT/SmallVector.h. It provides a STL-like
+ * vector class which uses a small allocation on the stack when the number of elements is small.
+ *
+ * This container is ideal for vectors which are allocated frequently, will more than likely only
+ * contain a few elements, and are allocated in places where performance is a concern. When the
+ * number of elements is small, storing these elements will not require a heap allocation but it can
+ * also grow to accommodate larger allocation needs. In other words, it effectively has memory
+ * allocation like std::array<T, N> until the number of elements exceeds N. Once the number of
+ * elements exceeds N, it turns into std::vector<T>.
+ *
+ * Reference: https://github.com/llvm/llvm-project/blob/main/llvm/include/llvm/ADT/SmallVector.h
+ */
+
 #include "lib/common/defines.hpp"
 
 #include <algorithm>

source/lib/rocprofiler/buffer.cpp

@@ -22,6 +22,7 @@
 
 #include "lib/rocprofiler/buffer.hpp"
 
+#include <glog/logging.h>
 #include <rocprofiler/rocprofiler.h>
 
 #include "lib/common/container/stable_vector.hpp"
@@ -35,6 +36,7 @@
 #include <atomic>
 #include <exception>
 #include <mutex>
+#include <random>
 #include <vector>
 
 namespace rocprofiler
@@ -51,8 +53,28 @@ get_buffers_mutex()
     static auto _v = std::mutex{};
     return _v;
 }
+
+uint64_t
+get_buffer_offset()
+{
+    static uint64_t _v = []() {
+        auto gen = std::mt19937{std::random_device{}()};
+        auto rng = std::uniform_int_distribution<uint64_t>{std::numeric_limits<uint8_t>::max(),
+                                                           std::numeric_limits<uint16_t>::max()};
+        return rng(gen);
+    }();
+    return _v;
+}
 }  // namespace
 
+bool
+is_valid_buffer_id(rocprofiler_buffer_id_t id)
+{
+    auto nbuffers = get_buffers().size();
+    auto offset   = get_buffer_offset();
+    return (id.handle >= offset && id.handle < (offset + nbuffers));
+}
+
 unique_buffer_vec_t&
 get_buffers()
 {
@@ -63,9 +85,15 @@ get_buffers()
 instance*
 get_buffer(rocprofiler_buffer_id_t buffer_id)
 {
-    for(auto& itr : get_buffers())
+    if(is_valid_buffer_id(buffer_id))
     {
-        if(itr && itr->buffer_id == buffer_id.handle) return itr.get();
+        for(auto& itr : get_buffers())
+        {
+            if(itr && itr->buffer_id == buffer_id.handle)
+            {
+                return itr.get();
+            }
+        }
     }
     return nullptr;
 }
@@ -81,7 +109,7 @@ allocate_buffer()
     auto _lk = std::unique_lock<std::mutex>{get_buffers_mutex()};
 
     // initial context identifier number
-    auto _idx = get_buffers().size();
+    auto _idx = get_buffer_offset() + get_buffers().size();
 
     // make space in registered
     get_buffers().emplace_back(nullptr);
@@ -93,6 +121,9 @@ allocate_buffer()
 
     if(!_cfg) return std::nullopt;
 
+    // set the buffer id value
+    _cfg_v->buffer_id = _idx;
+
     return rocprofiler_buffer_id_t{_idx};
 }
 
@@ -101,9 +132,11 @@ flush(rocprofiler_buffer_id_t buffer_id, bool wait)
 {
     if(registration::get_fini_status() > 0) return ROCPROFILER_STATUS_SUCCESS;
 
-    if(buffer_id.handle >= get_buffers().size()) return ROCPROFILER_STATUS_ERROR_BUFFER_NOT_FOUND;
+    auto offset = get_buffer_offset();
 
-    auto& buff = get_buffers().at(buffer_id.handle);
+    if(!is_valid_buffer_id(buffer_id)) return ROCPROFILER_STATUS_ERROR_BUFFER_NOT_FOUND;
+
+    auto* buff = get_buffer(buffer_id);
 
     auto* task_group =
         internal_threading::get_task_group(rocprofiler_callback_thread_t{buff->task_group_id});
@@ -115,8 +148,8 @@ flush(rocprofiler_buffer_id_t buffer_id, bool wait)
 
     auto idx = buff->buffer_idx++;
 
-    auto _task = [buffer_id, idx]() {
-        auto& buff_v          = get_buffers().at(buffer_id.handle);
+    auto _task = [buffer_id, idx, offset]() {
+        auto& buff_v          = get_buffers().at(buffer_id.handle - offset);
         auto& buff_internal_v = buff_v->get_internal_buffer(idx);
 
         if(!buff_internal_v.is_empty())
@@ -179,11 +212,20 @@ rocprofiler_create_buffer(rocprofiler_context_id_t        context,
     if(rocprofiler::registration::get_init_status() > -1)
         return ROCPROFILER_STATUS_ERROR_CONFIGURATION_LOCKED;
 
+    auto* existing_buff = rocprofiler::buffer::get_buffer(*buffer_id);
+    if(existing_buff)
+    {
+        LOG(ERROR) << "buffer (handle=" << buffer_id->handle
+                   << ") already allocated: handle=" << existing_buff->buffer_id;
+        return ROCPROFILER_STATUS_ERROR_SERVICE_ALREADY_CONFIGURED;
+    }
+
     auto opt_buff_id = rocprofiler::buffer::allocate_buffer();
     if(!opt_buff_id) return ROCPROFILER_STATUS_ERROR_BUFFER_NOT_FOUND;
-    *buffer_id = *opt_buff_id;
+    buffer_id->handle = opt_buff_id->handle;
 
-    auto& buff = rocprofiler::buffer::get_buffers().at(opt_buff_id->handle);
+    auto& buff = rocprofiler::buffer::get_buffers().at(opt_buff_id->handle -
+                                                       rocprofiler::buffer::get_buffer_offset());
 
     // allocate the buffers. if it is lossless, we allocate a second buffer to store data while
     // other buffer is being flushed
@@ -195,7 +237,8 @@ rocprofiler_create_buffer(rocprofiler_context_id_t        context,
     buff->callback      = callback;
     buff->callback_data = callback_data;
     buff->context_id    = context.handle;
-    buff->buffer_idx    = buffer_id->handle;
+    buff->buffer_id     = buffer_id->handle;
+    buff->buffer_idx    = 0;
 
     return ROCPROFILER_STATUS_SUCCESS;
 }
@@ -209,10 +252,12 @@ rocprofiler_flush_buffer(rocprofiler_buffer_id_t buffer_id)
 rocprofiler_status_t
 rocprofiler_destroy_buffer(rocprofiler_buffer_id_t buffer_id)
 {
-    if(buffer_id.handle >= rocprofiler::buffer::get_buffers().size())
+    if(!rocprofiler::buffer::is_valid_buffer_id(buffer_id))
         return ROCPROFILER_STATUS_ERROR_BUFFER_NOT_FOUND;
 
-    auto& buff = rocprofiler::buffer::get_buffers().at(buffer_id.handle);
+    auto  offset  = rocprofiler::buffer::get_buffer_offset();
+    auto& buffers = rocprofiler::buffer::get_buffers();
+    auto& buff    = buffers.at(buffer_id.handle - offset);
 
     // buffer is currently being flushed or destroyed
     if(buff->syncer.test_and_set()) return ROCPROFILER_STATUS_ERROR_BUFFER_BUSY;

source/lib/rocprofiler/buffer.hpp

@@ -45,12 +45,12 @@ struct instance
 
     mutable std::array<buffer_t, 2> buffers       = {};
     mutable std::atomic_flag        syncer        = ATOMIC_FLAG_INIT;
-    mutable std::atomic<uint32_t>   buffer_idx    = {};
+    mutable std::atomic<uint32_t>   buffer_idx    = {};  // array index
     mutable std::atomic<uint64_t>   drop_count    = {};
     uint64_t                        watermark     = 0;
-    uint64_t                        context_id    = 0;
-    uint64_t                        buffer_id     = 0;
-    uint64_t                        task_group_id = 0;
+    uint64_t                        context_id    = 0;  // rocprofiler_context_id_t value
+    uint64_t                        buffer_id     = 0;  // rocprofiler_buffer_id_t value
+    uint64_t                        task_group_id = 0;  // thread-pool assignment
     rocprofiler_buffer_tracing_cb_t callback      = nullptr;
     void*                           callback_data = nullptr;
     rocprofiler_buffer_policy_t     policy        = ROCPROFILER_BUFFER_POLICY_NONE;
@@ -65,6 +65,9 @@ struct instance
 using unique_buffer_vec_t =
     common::container::stable_vector<allocator::unique_static_ptr_t<instance>, 4>;
 
+bool
+is_valid_buffer_id(rocprofiler_buffer_id_t id);
+
 std::optional<rocprofiler_buffer_id_t>
 allocate_buffer();
 

source/lib/rocprofiler/hsa/hsa.cpp

@@ -336,21 +336,17 @@ hsa_api_impl<Idx>::functor(Args&&... args)
             assert(itr.ctx->buffered_tracer);
             auto buffer_id =
                 itr.ctx->buffered_tracer->buffer_data.at(info_type::buffered_domain_idx);
-            for(auto& bitr : buffer::get_buffers())
+            auto buffer_v = buffer::get_buffer(buffer_id);
+            if(buffer_v && buffer_v->context_id == itr.ctx->context_idx &&
+               buffer_v->buffer_id == buffer_id.handle)
             {
-                if(bitr && bitr->context_id == itr.ctx->context_idx &&
-                   bitr->buffer_id == buffer_id.handle)
-                {
-                    // make copy of record
-                    auto record_v = buffer_record;
-                    // update the record with the correlation
-                    record_v.correlation_id.external = itr.external_correlation;
+                // make copy of record
+                auto record_v = buffer_record;
+                // update the record with the correlation
+                record_v.correlation_id.external = itr.external_correlation;
 
-                    bitr->emplace(ROCPROFILER_BUFFER_CATEGORY_TRACING,
-                                  info_type::buffered_domain_idx,
-                                  record_v);
-                    break;
-                }
+                buffer_v->emplace(
+                    ROCPROFILER_BUFFER_CATEGORY_TRACING, info_type::buffered_domain_idx, record_v);
             }
         }
     }

source/lib/rocprofiler/hsa/queue.cpp

@@ -175,9 +175,9 @@ AsyncSignalHandler(hsa_signal_value_t /*signal_v*/, void* data)
                                            dispatch_packet.grid_size_y,
                                            dispatch_packet.grid_size_z}};
 
-                    _buffer->emplace(ROCPROFILER_BUFFER_CATEGORY_TRACING,
-                                     ROCPROFILER_BUFFER_TRACING_KERNEL_DISPATCH,
-                                     record);
+                    CHECK_NOTNULL(_buffer)->emplace(ROCPROFILER_BUFFER_CATEGORY_TRACING,
+                                                    ROCPROFILER_BUFFER_TRACING_KERNEL_DISPATCH,
+                                                    record);
                 }
             }
 
@@ -195,9 +195,9 @@ AsyncSignalHandler(hsa_signal_value_t /*signal_v*/, void* data)
                         _queue_id,
                         _kern_id};
 
-                    _buffer->emplace(ROCPROFILER_BUFFER_CATEGORY_TRACING,
-                                     ROCPROFILER_BUFFER_TRACING_MEMORY_COPY,
-                                     record);
+                    CHECK_NOTNULL(_buffer)->emplace(ROCPROFILER_BUFFER_CATEGORY_TRACING,
+                                                    ROCPROFILER_BUFFER_TRACING_MEMORY_COPY,
+                                                    record);
                 }
             }
         }

source/lib/rocprofiler/internal_threading.cpp

@@ -292,14 +292,13 @@ rocprofiler_assign_callback_thread(rocprofiler_buffer_id_t       buffer_id,
     if(cb_thread_id.handle >= rocprofiler::internal_threading::get_task_groups()->size())
         return ROCPROFILER_STATUS_ERROR_THREAD_NOT_FOUND;
 
-    for(auto& bitr : rocprofiler::buffer::get_buffers())
+    auto* buff_v = rocprofiler::buffer::get_buffer(buffer_id);
+    if(buff_v)
     {
-        if(bitr && bitr->buffer_id == buffer_id.handle)
-        {
-            bitr->task_group_id = cb_thread_id.handle;
-            return ROCPROFILER_STATUS_SUCCESS;
-        }
+        buff_v->task_group_id = cb_thread_id.handle;
+        return ROCPROFILER_STATUS_SUCCESS;
     }
+
     return ROCPROFILER_STATUS_ERROR_BUFFER_NOT_FOUND;
 }
 }

source/lib/rocprofiler/registration.cpp

@@ -21,6 +21,7 @@
 // SOFTWARE.
 
 #include "lib/rocprofiler/registration.hpp"
+#include "lib/common/environment.hpp"
 #include "lib/rocprofiler/agent.hpp"
 #include "lib/rocprofiler/allocator.hpp"
 #include "lib/rocprofiler/context/context.hpp"
@@ -171,9 +172,38 @@ find_clients()
                                          rocprofiler_client_id_t{nullptr, _prio}});
     }
 
-    if(!rocprofiler_configure && !get_forced_configure())
+    auto get_env_libs = []() {
+        auto       val       = common::get_env("ROCP_TOOL_LIBRARIES", std::string{});
+        auto       val_arr   = std::vector<std::string>{};
+        size_t     pos       = 0;
+        const auto delimiter = std::string_view{":"};
+        auto       token     = std::string{};
+
+        if(val.empty())
+        {
+            // do nothing
+        }
+        else if(val.find(delimiter) == std::string::npos)
+        {
+            val_arr.emplace_back(val);
+        }
+        else
+        {
+            while((pos = val.find(delimiter)) != std::string::npos)
+            {
+                token = val.substr(0, pos);
+                if(!token.empty()) val_arr.emplace_back(token);
+                val.erase(0, pos + delimiter.length());
+            }
+        }
+        return val_arr;
+    };
+
+    auto env = get_env_libs();
+
+    if(!rocprofiler_configure && !get_forced_configure() && env.empty())
     {
-        LOG(ERROR) << "no rocprofiler_configure function found";
+        LOG(ERROR) << "no rocprofiler_configure function(s) found";
         return data;
     }
 
@@ -201,6 +231,13 @@ find_clients()
         decltype(::rocprofiler_configure)* _sym = nullptr;
         *(void**) (&_sym)                       = dlsym(handle, "rocprofiler_configure");
 
+        // symbol not found
+        if(!_sym)
+        {
+            LOG(INFO) << "|_" << itr << " did not contain rocprofiler_configure symbol";
+            continue;
+        }
+
         // skip the configure function that was forced
         if(_sym == get_forced_configure())
         {
@@ -210,12 +247,6 @@ find_clients()
             continue;
         }
 
-        if(!_sym)
-        {
-            LOG(INFO) << "|_" << itr << " did not contain rocprofiler_configure symbol";
-            continue;
-        }
-
         if(_sym == &rocprofiler_configure && data.size() == 1)
         {
             data.front().name                    = itr;
@@ -236,6 +267,38 @@ find_clients()
         }
     }
 
+    if(!env.empty())
+    {
+        for(const auto& itr : env)
+        {
+            void* handle = dlopen(itr.c_str(), RTLD_GLOBAL | RTLD_LAZY);
+            LOG_IF(ERROR, handle == nullptr) << "error dlopening " << itr;
+
+            for(const auto& ditr : data)
+            {
+                if(ditr.dlhandle && ditr.dlhandle == handle)
+                {
+                    handle = nullptr;
+                    break;
+                }
+            }
+
+            if(handle)
+            {
+                decltype(::rocprofiler_configure)* _sym = nullptr;
+                *(void**) (&_sym)                       = dlsym(handle, "rocprofiler_configure");
+
+                uint32_t _prio = priority_offset + data.size();
+                data.emplace_back(client_library{itr,
+                                                 handle,
+                                                 _sym,
+                                                 nullptr,
+                                                 rocprofiler_client_id_t{nullptr, _prio},
+                                                 rocprofiler_client_id_t{nullptr, _prio}});
+            }
+        }
+    }
+
     LOG(ERROR) << __FUNCTION__ << " found " << data.size() << " clients";
 
     return data;
@@ -479,16 +542,26 @@ initialize()
 void
 finalize()
 {
-    if(get_fini_status() != 0) return;
+    if(get_fini_status() != 0)
+    {
+        LOG(INFO) << "ignoring finalization request (value=" << get_fini_status() << ")";
+        return;
+    }
 
     static auto _sync = std::atomic_flag{};
-    if(_sync.test_and_set()) return;
+    if(_sync.test_and_set())
+    {
+        LOG(INFO) << "ignoring finalization request [already finalized] (value="
+                  << get_fini_status() << ")";
+        return;
+    }
     // above returns true for all invocations after the first one
 
+    LOG(INFO) << "finalizing rocprofiler (value=" << get_fini_status() << ")";
+
     static auto _once = std::once_flag{};
     std::call_once(_once, []() {
         set_fini_status(-1);
-        ::hsa_shut_down();
         hsa::code_object_shutdown();
         if(get_init_status() > 0)
         {
@@ -560,6 +633,11 @@ rocprofiler_set_api_table(const char* name,
     }
     else if(std::string_view{name} == "hsa")
     {
+        // this is a slight hack due to a hsa-runtime bug with rocprofiler-register which
+        // causes it to register the API table twice when HSA_TOOL_LIB is set to this
+        // rocprofiler library. Fixed in Gerrit review 961592.
+        setenv("HSA_TOOLS_ROCPROFILER_V1_TOOLS", "0", 0);
+
         // pass to hsa init
         LOG_IF(ERROR, num_tables > 1)
             << " rocprofiler expected HSA library to pass 1 API table, not " << num_tables;
@@ -623,4 +701,12 @@ OnLoad(HsaApiTable*       table,
 
     return true;
 }
+
+void
+OnUnload()
+{
+    LOG(INFO) << "Unloading hsa-runtime...";
+    ::rocprofiler::registration::finalize();
+    LOG(INFO) << "Finalization complete.";
+}
 }

source/lib/rocprofiler/registration.hpp

@@ -40,6 +40,9 @@ OnLoad(HsaApiTable*       table,
        uint64_t           failed_tool_count,
        const char* const* failed_tool_names) ROCPROFILER_PUBLIC_API;
 
+void
+OnUnload() ROCPROFILER_PUBLIC_API;
+
 // this is the "hidden" function that rocprofiler-register invokes to pass
 // the API tables to rocprofiler
 int

source/lib/rocprofiler/tests/buffer.cpp

@@ -45,13 +45,16 @@ TEST(rocprofiler_lib, buffer)
     auto buffer_id = buffer::allocate_buffer();
 
     EXPECT_TRUE(buffer_id) << "failed to allocate buffer";
+    EXPECT_GT(buffer_id->handle, 0);
+    EXPECT_TRUE(buffer::is_valid_buffer_id(*buffer_id)) << "id=" << buffer_id->handle;
     ASSERT_EQ(buffer::get_buffers().size(), 1) << "incorrect number of buffers created";
 
     // get pointer to buffer
-    auto* buffer_v      = buffer::get_buffer(*buffer_id);
-    buffer_v->watermark = common::units::get_page_size();
-    ASSERT_NE(buffer_v, nullptr) << "get_buffer returned a nullptr";
+    auto* buffer_v = buffer::get_buffer(*buffer_id);
+    ASSERT_NE(buffer_v, nullptr) << "get_buffer returned a nullptr. id=" << buffer_id->handle;
+    EXPECT_EQ(buffer_v->buffer_id, buffer_id->handle);
 
+    buffer_v->watermark = common::units::get_page_size();
     {
         auto records = buffer_v->get_internal_buffer().get_record_headers();
         EXPECT_EQ(records.size(), 0);

source/scripts/run-ci.py

@@ -241,7 +241,7 @@ def parse_cdash_args(args):
         "-c",
         "--coverage",
         help="Enable code coverage",
-        choices=("all", "unittests", "samples"),
+        choices=("all", "tests", "samples"),
         type=str,
         default=None,
     )
@@ -376,8 +376,8 @@ def parse_args(args=None):
         cmake_args += ["-DROCPROFILER_BUILD_CODECOV=ON"]
         if cdash_args.coverage == "samples":
             ctest_args += ["-L", "samples"]
-        elif cdash_args.coverage == "unittests":
-            ctest_args += ["-L", "unittests"]
+        elif cdash_args.coverage == "tests":
+            ctest_args += ["-L", "tests"]
 
     if cdash_args.linter == "clang-tidy":
         cmake_args += ["-DROCPROFILER_ENABLE_CLANG_TIDY=ON"]
@@ -438,6 +438,7 @@ if __name__ == "__main__":
         dashboard_args.append(f"{args.mode}{itr}")
 
     try:
+        ctest_args += ["--no-tests=error"]
         if not args.quiet and len(ctest_args) == 0:
             ctest_args = ["--output-on-failure", "-V"]