Buffering: initial implementation and tests (#20)

* Update source/lib/common - CMakeLists.txt - less verbose - rocprofiler-common-library uses rocprofiler-headers target - mpl.hpp - metaprogramming header with type_list, size_of, index_of, and is_one_of - record_header_buffer.{hpp,cpp} - wrapper class around atomic_ring_buffer and vector of rocprofiler_record_header_t - atomic_ring_buffer.{hpp,cpp} - request function accepts wrap param when overwritting is not desirable - can_clear member function - clear member function for rewinding write pointer to start of buffer - containers/CMakeLists.txt - include record_header_buffer.{hpp,cpp} in build target * Update source/lib/tests: Buffering tests - Added buffering tests. See comments in code for description * atomic_ring_buffer -> ring_buffer - remove ring_buffer implementation - rename atomic_ring_buffer to ring_buffer * atomic_ring_buffer -> ring_buffer - remove ring_buffer implementation - rename atomic_ring_buffer to ring_buffer * Update record_header_buffer - lock, unlock, is_locked, clear, save, and load member functions * Buffering tests - add buffer test for save/load capability * Update rocprofiler_memcheck.cmake - fix erroneous spaces causing incorrect string evaluation * Update ring_buffer - fix exception message * undef HIP_PROF_API - make sure HIP_PROF_API is undefined before including hip_runtime.h - avoid directly including hip/hip_runtime.h * Update rocprofiler_config_interfaces - remove stale preprocessor defines that are from old rocprofiler/roctracer - HIP_PROF_HIP_API_STRING=1 - PROF_API_IMPL=1 * Update run-ci.py - fix paths to suppression files - improve printing logs to console in github actions * Update buffering implementation - remove support for using malloc instead of mmap in ring_buffer - provide some info functions in record_header_buffer - improve the testing of the save-load buffer test * Update run-ci.py - fix CTEST_CUSTOM_COVERAGE_EXCLUDE * Update hip/api_args.h - remove undef HIP_PROF_API * Update buffering-save-load.cpp - updated comments * Update record_header_buffer - default ctor - allocate member function - is_allocated member function * Update buffering-save-load.cpp - tweaked usage of record_header_buffer to delay allocation [ROCm/rocprofiler-sdk commit: b12ef4a75e]
2023-08-30 11:34:03 -05:00
parent d4a977349c
commit ccd154b74c
20 changed files with 1525 additions and 1063 deletions
@@ -1 +1,4 @@
-
+#
+#
+#
+add_subdirectory(buffering)
@@ -0,0 +1,23 @@
+#
+#
+#
+project(rocprofiler-tests-buffering LANGUAGES C CXX)
+
+include(GoogleTest)
+
+set(buffering_sources buffering-serial.cpp buffering-parallel.cpp buffering-save-load.cpp)
+
+add_executable(buffering-test)
+target_sources(buffering-test PRIVATE ${buffering_sources})
+target_link_libraries(
+    buffering-test
+    PRIVATE rocprofiler::rocprofiler-headers rocprofiler::rocprofiler-common-library
+            GTest::gtest GTest::gtest_main)
+
+gtest_add_tests(
+    TARGET buffering-test
+    SOURCES ${buffering_sources}
+    TEST_LIST buffering-test_TESTS
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+
+set_tests_properties(${buffering-tests_TESTS} PROPERTIES TIMEOUT 45 LABELS "unittests")
@@ -0,0 +1,217 @@
+// MIT License
+//
+// Copyright (c) 2023 ROCm Developer Tools
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "buffering.hpp"
+#include "lib/common/container/record_header_buffer.hpp"
+#include "lib/common/mpl.hpp"
+#include "lib/common/units.hpp"
+
+#include <gtest/gtest.h>
+#include <pthread.h>
+
+#include <cstdint>
+#include <cstdlib>
+#include <limits>
+#include <typeinfo>
+#include <utility>
+
+namespace
+{
+namespace test  = ::rocprofiler::test;
+namespace units = ::rocprofiler::common::units;
+namespace mpl   = ::rocprofiler::common::mpl;
+
+using record_header_buffer_t = rocprofiler::common::container::record_header_buffer;
+
+// this function returns a random array of values specific to template instantiation
+template <typename Tp, size_t N>
+auto&
+get_generated_array()
+{
+    static auto _value = []() {
+        auto _v = test::raw_array<Tp, N>{};
+        test::generate(_v, Tp{0}, std::numeric_limits<Tp>::max());
+        return _v;
+    }();
+    return _value;
+}
+
+// these are the array size variants. we use the units to scale up
+// but technically the data size of the raw_array will be multiplied
+// by sizeof(Tp)
+constexpr auto test_data_sizes = std::index_sequence<1 * units::byte,
+                                                     2 * units::byte,
+                                                     3 * units::byte,
+                                                     4 * units::byte,
+                                                     8 * units::byte,
+                                                     16 * units::kilobyte,
+                                                     20 * units::kilobyte,
+                                                     24 * units::kilobyte,
+                                                     32 * units::kilobyte,
+                                                     56 * units::kilobyte,
+                                                     64 * units::kilobyte,
+                                                     91 * units::kilobyte,
+                                                     128 * units::kilobyte,
+                                                     387 * units::kilobyte,
+                                                     693 * units::kilobyte,
+                                                     2 * units::megabyte>{};
+
+// this is the list of array data types we will generate. Effectively, there
+// will be one raw array for each combination of these types and the test data sizes
+// (i.e. there will be unique 160 arrays of different types and sizes)
+using test_data_types = mpl::type_list<int8_t,
+                                       uint8_t,
+                                       int16_t,
+                                       uint16_t,
+                                       int32_t,
+                                       uint32_t,
+                                       int64_t,
+                                       uint64_t,
+                                       float,
+                                       double>;
+
+// this function creates a thread for each data size for a given type.
+// all threads are detached and will wait at the first barrier until all
+// threads have reached it, race to emplace their data in the shared
+// buffer and then wait at the second barrier until all the threads have
+// emplacing the data and the main thread has also reached the second
+// barrier.
+template <typename Tp, size_t... Idx>
+void
+launch_threads(record_header_buffer_t& _buf,
+               pthread_barrier_t&      _race_barrier,
+               pthread_barrier_t&      _done_barrier,
+               std::index_sequence<Idx...>)
+{
+    auto _launch =
+        [](record_header_buffer_t* _buf_v, auto* _race_barrier_v, auto* _done_barrier_v, auto* _v) {
+            pthread_barrier_wait(_race_barrier_v);
+            EXPECT_TRUE(_buf_v->emplace(*_v));
+            pthread_barrier_wait(_done_barrier_v);
+        };
+    (std::thread{_launch, &_buf, &_race_barrier, &_done_barrier, &get_generated_array<Tp, Idx>()}
+         .detach(),
+     ...);
+}
+
+// expansion for each type
+template <typename... Tp, size_t... Idx>
+void
+launch_threads(record_header_buffer_t& _buf,
+               pthread_barrier_t&      _race_barrier,
+               pthread_barrier_t&      _done_barrier,
+               mpl::type_list<Tp...>,
+               std::index_sequence<Idx...> _seq)
+{
+    (launch_threads<Tp>(_buf, _race_barrier, _done_barrier, _seq), ...);
+}
+
+// computes the size of every raw_array size for a given type
+template <typename Tp, size_t... Idx>
+constexpr size_t get_data_size(std::index_sequence<Idx...>)
+{
+    size_t _v = 0;
+    ((_v += sizeof(get_generated_array<Tp, Idx>())), ...);
+    return _v;
+}
+
+// expansion for each type
+template <typename... Tp, size_t... Idx>
+constexpr size_t
+get_data_size(mpl::type_list<Tp...>, std::index_sequence<Idx...> _seq)
+{
+    size_t _v = 0;
+    ((_v += get_data_size<Tp>(_seq)), ...);
+    return _v;
+}
+
+// validates that the raw array extracted out of the buffer is equal
+// to the raw array that was placed in the buffer
+template <typename Tp, size_t N>
+void
+validate(const std::vector<rocprofiler_record_header_t*>& _headers)
+{
+    using data_type = test::raw_array<Tp, N>;
+    auto& _ref_data = get_generated_array<Tp, N>();
+    for(auto* itr : _headers)
+    {
+        if(itr->kind == typeid(data_type).hash_code())
+        {
+            auto* _data = static_cast<data_type*>(itr->payload);
+            EXPECT_EQ(_ref_data, *_data);
+        }
+    }
+}
+
+// expansion for every raw array size for a given data type
+template <typename Tp, size_t... Idx>
+void
+validate(const std::vector<rocprofiler_record_header_t*>& _headers, std::index_sequence<Idx...>)
+{
+    (validate<Tp, Idx>(_headers), ...);
+}
+
+// expansion for each raw array type
+template <typename... Tp, size_t... Idx>
+void
+validate(const std::vector<rocprofiler_record_header_t*>& _headers,
+         mpl::type_list<Tp...>,
+         std::index_sequence<Idx...> _seq)
+{
+    (validate<Tp>(_headers, _seq), ...);
+}
+}  // namespace
+
+TEST(buffering, parallel)
+{
+    // this test launches 160 threads, each with a randomly generated array of data
+    // and has them contend for emplacing their data in the same buffer. The purpose
+    // of this test is to validate that multiple threads can write to the same
+    // (lock-free) buffer without any data corruption or loss.
+
+    constexpr auto num_variants = test_data_types::size() * test_data_sizes.size();
+    constexpr auto data_size    = get_data_size(test_data_types{}, test_data_sizes);
+
+    EXPECT_EQ(num_variants, 160);
+
+    // make a buffer large enough to hold all the data we generate
+    auto _buffer = record_header_buffer_t{data_size};
+
+    // create a barrier that all child threads will wait and then race to enqueue their data in the
+    // buffer i.e., we want to maximize contention on inserting into buffer
+    auto _data_race_barrier = pthread_barrier_t{};
+    pthread_barrier_init(&_data_race_barrier, nullptr, num_variants);
+
+    // a barrier to signal that all threads have completed placing their data in the buffer
+    auto _emplaced_barrier = pthread_barrier_t{};
+    pthread_barrier_init(&_emplaced_barrier, nullptr, num_variants + 1);
+
+    // launch 160 threads
+    launch_threads(
+        _buffer, _data_race_barrier, _emplaced_barrier, test_data_types{}, test_data_sizes);
+
+    // wait for all the threads to complete
+    pthread_barrier_wait(&_emplaced_barrier);
+
+    // verify the data pulled out the buffer matches the data put in by the threads
+    validate(_buffer.get_record_headers(), test_data_types{}, test_data_sizes);
+}
@@ -0,0 +1,273 @@
+// MIT License
+//
+// Copyright (c) 2023 ROCm Developer Tools
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "buffering.hpp"
+#include "lib/common/container/record_header_buffer.hpp"
+#include "lib/common/mpl.hpp"
+#include "lib/common/units.hpp"
+
+#include <gtest/gtest.h>
+#include <pthread.h>
+
+#include <cstdint>
+#include <cstdlib>
+#include <fstream>
+#include <limits>
+#include <typeinfo>
+#include <utility>
+
+namespace
+{
+namespace test  = ::rocprofiler::test;
+namespace units = ::rocprofiler::common::units;
+namespace mpl   = ::rocprofiler::common::mpl;
+
+using record_header_buffer_t = rocprofiler::common::container::record_header_buffer;
+
+// this function returns a random array of values specific to template instantiation
+template <typename Tp, size_t N>
+auto&
+get_generated_array()
+{
+    static auto _value = []() {
+        auto _v = test::raw_array<Tp, N>{};
+        test::generate(_v, Tp{0}, std::numeric_limits<Tp>::max());
+        return _v;
+    }();
+    return _value;
+}
+
+// these are the array size variants. we use the units to scale up
+// but technically the data size of the raw_array will be multiplied
+// by sizeof(Tp)
+constexpr auto test_data_sizes = std::index_sequence<1 * units::byte,
+                                                     2 * units::byte,
+                                                     3 * units::byte,
+                                                     4 * units::byte,
+                                                     8 * units::byte,
+                                                     16 * units::kilobyte,
+                                                     20 * units::kilobyte,
+                                                     24 * units::kilobyte,
+                                                     32 * units::kilobyte,
+                                                     56 * units::kilobyte,
+                                                     64 * units::kilobyte,
+                                                     91 * units::kilobyte>{};
+
+// this is the list of array data types we will generate. Effectively, there
+// will be one raw array for each combination of these types and the test data sizes
+// (i.e. there will be unique 160 arrays of different types and sizes)
+using test_data_types = mpl::type_list<int8_t,
+                                       uint8_t,
+                                       int16_t,
+                                       uint16_t,
+                                       int32_t,
+                                       uint32_t,
+                                       int64_t,
+                                       uint64_t,
+                                       float,
+                                       double>;
+
+// this function creates a thread for each data size for a given type.
+// all threads are detached and will wait at the first barrier until all
+// threads have reached it, race to emplace their data in the shared
+// buffer and then wait at the second barrier until all the threads have
+// emplacing the data and the main thread has also reached the second
+// barrier.
+template <typename Tp, size_t... Idx>
+void
+launch(record_header_buffer_t* _buf, pthread_barrier_t* _done_barrier, std::index_sequence<Idx...>)
+{
+    auto _launch = [](record_header_buffer_t* _buf_v, auto* _v) {
+        EXPECT_TRUE(_buf_v->emplace(*_v));
+    };
+    (_launch(_buf, &get_generated_array<Tp, Idx>()), ...);
+    pthread_barrier_wait(_done_barrier);
+}
+
+// expansion for each type
+template <typename... Tp, size_t... Idx>
+void
+launch_threads(record_header_buffer_t& _buf,
+               pthread_barrier_t&      _done_barrier,
+               mpl::type_list<Tp...>,
+               std::index_sequence<Idx...> _seq)
+{
+    ((std::thread{[_seq](auto* _buf_v, auto* _barrier_v) { launch<Tp>(_buf_v, _barrier_v, _seq); },
+                  &_buf,
+                  &_done_barrier}
+          .detach()),
+     ...);
+}
+
+// computes the size of every raw_array size for a given type
+template <typename Tp, size_t... Idx>
+constexpr size_t get_data_size(std::index_sequence<Idx...>)
+{
+    size_t _v = 0;
+    ((_v += sizeof(get_generated_array<Tp, Idx>())), ...);
+    return _v;
+}
+
+// expansion for each type
+template <typename... Tp, size_t... Idx>
+constexpr size_t
+get_data_size(mpl::type_list<Tp...>, std::index_sequence<Idx...> _seq)
+{
+    size_t _v = 0;
+    ((_v += get_data_size<Tp>(_seq)), ...);
+    return _v;
+}
+
+// validates that the raw array extracted out of the buffer is equal
+// to the raw array that was placed in the buffer
+template <typename Tp, size_t N>
+void
+validate(const std::vector<rocprofiler_record_header_t*>& _headers)
+{
+    using data_type = test::raw_array<Tp, N>;
+    auto& _ref_data = get_generated_array<Tp, N>();
+    for(auto* itr : _headers)
+    {
+        if(itr->kind == typeid(data_type).hash_code())
+        {
+            auto* _data = static_cast<data_type*>(itr->payload);
+            ASSERT_TRUE(_data != nullptr);
+            EXPECT_EQ(_ref_data, *_data);
+        }
+    }
+}
+
+// expansion for every raw array size for a given data type
+template <typename Tp, size_t... Idx>
+void
+validate(const std::vector<rocprofiler_record_header_t*>& _headers, std::index_sequence<Idx...>)
+{
+    (validate<Tp, Idx>(_headers), ...);
+}
+
+// expansion for each raw array type
+template <typename... Tp, size_t... Idx>
+void
+validate(const std::vector<rocprofiler_record_header_t*>& _headers,
+         mpl::type_list<Tp...>,
+         std::index_sequence<Idx...> _seq)
+{
+    (validate<Tp>(_headers, _seq), ...);
+}
+}  // namespace
+
+TEST(buffering, save_load)
+{
+    // this test launches 10 threads for each of the data types in test_data_types. Each thread
+    // randomly generates 12 array of data of differing sizes and contends with the other threads
+    // for emplacing the data in the same buffer. The purpose of this test is test the thread-safety
+    // in a slightly different way, save it to a file backing, clear it, restore it from the file,
+    // and move it to another object and ensure that the data after the save + load + move matches
+    // the original data placed into the buffer without any data corruption or loss
+
+    constexpr auto num_variants = test_data_types::size() * test_data_sizes.size();
+    constexpr auto data_size    = get_data_size(test_data_types{}, test_data_sizes);
+
+    EXPECT_EQ(num_variants, 120);
+
+    // make a buffer large enough to hold all the data we generate
+    auto _buffer = record_header_buffer_t{};
+
+    EXPECT_FALSE(_buffer.is_allocated());
+    EXPECT_EQ(_buffer.size(), 0);
+    EXPECT_EQ(_buffer.count(), 0);
+    EXPECT_EQ(_buffer.free(), 0);
+    EXPECT_EQ(_buffer.capacity(), 0);
+    EXPECT_TRUE(_buffer.is_empty());
+    EXPECT_TRUE(_buffer.is_full());
+
+    // allocate the buffer
+    ASSERT_TRUE(_buffer.allocate(data_size)) << "buffer failed to allocate";
+
+    EXPECT_EQ(_buffer.size(), 0);
+    EXPECT_EQ(_buffer.count(), 0);
+    EXPECT_GE(_buffer.free(), data_size);
+    EXPECT_GE(_buffer.capacity(), data_size);
+    EXPECT_TRUE(_buffer.is_empty());
+    EXPECT_FALSE(_buffer.is_full());
+
+    // a barrier to signal that all threads have completed placing their data in the buffer
+    auto _emplaced_barrier = pthread_barrier_t{};
+    pthread_barrier_init(&_emplaced_barrier, nullptr, test_data_types::size() + 1);
+
+    // launch 160 threads
+    launch_threads(_buffer, _emplaced_barrier, test_data_types{}, test_data_sizes);
+
+    // wait for all the threads to complete
+    pthread_barrier_wait(&_emplaced_barrier);
+
+    // verify the data, at a high-level is correct
+    EXPECT_EQ(_buffer.size(), num_variants);
+    EXPECT_EQ(_buffer.count(), data_size);
+    EXPECT_GE(_buffer.free(), 0);
+    EXPECT_GE(_buffer.capacity(), data_size);
+    EXPECT_FALSE(_buffer.is_empty());
+
+    // verify the data pulled out the buffer matches the data put in
+    validate(_buffer.get_record_headers(), test_data_types{}, test_data_sizes);
+
+    // save the data to a binary file and clear the buffer so it can "receive" new data (in theory)
+    {
+        auto _ofs = std::fstream{};
+        _ofs.open("buffer-save-load.dat", std::ios::out);
+        _buffer.save(_ofs);
+        EXPECT_EQ(_buffer.clear(), num_variants);
+    }
+
+    // verify that the buffer is empty
+    EXPECT_EQ(_buffer.get_record_headers().size(), 0) << "buffer was not cleared properly";
+
+    // load the data back from the binary file
+    {
+        auto _ifs = std::fstream{};
+        _ifs.open("buffer-save-load.dat", std::ios::in);
+        _buffer.load(_ifs);
+    }
+
+    // verify that, at a high level, all the data was preserved
+    ASSERT_EQ(_buffer.get_record_headers().size(), num_variants)
+        << "buffer was not saved/loaded properly";
+
+    // verify the data is entirely correct
+    validate(_buffer.get_record_headers(), test_data_types{}, test_data_sizes);
+
+    // move the data into another instance of record_header_buffer_t
+    auto _buffer_v = record_header_buffer_t{std::move(_buffer)};
+
+    // make sure the move emptied out the old object and populated the new object
+    ASSERT_EQ(_buffer.get_record_headers().size(), 0) << "buffer was not moved properly";
+    ASSERT_EQ(_buffer_v.get_record_headers().size(), num_variants)
+        << "buffer was not moved properly";
+
+    // validate the data in the new object
+    // verify the data pulled out the buffer matches the data put in by the threads
+    validate(_buffer_v.get_record_headers(), test_data_types{}, test_data_sizes);
+
+    // make sure reset works when empty and when full
+    EXPECT_EQ(_buffer.reset(), 0) << "buffer should be empty after move";
+    EXPECT_EQ(_buffer_v.reset(), num_variants);
+}
@@ -0,0 +1,172 @@
+// MIT License
+//
+// Copyright (c) 2023 ROCm Developer Tools
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#include "buffering.hpp"
+#include "lib/common/container/record_header_buffer.hpp"
+
+#include <gtest/gtest.h>
+#include <pthread.h>
+
+#include <cstdint>
+#include <cstdlib>
+#include <random>
+#include <typeinfo>
+
+namespace
+{
+namespace test = ::rocprofiler::test;
+
+using uint_raw_array_t       = test::raw_array<uint64_t, 32>;
+using flt_raw_array_t        = test::raw_array<double, 64>;
+using record_header_buffer_t = rocprofiler::common::container::record_header_buffer;
+
+// generates an array with random data
+template <typename Tp, size_t N>
+auto
+generate_array(Tp _low = 0UL, Tp _high = 1000UL)
+{
+    auto _v = test::raw_array<Tp, N>{};
+    test::generate(_v, _low, _high);
+    return _v;
+}
+
+// pulls out a raw array of the given type and puts back into a vector
+template <typename Tp>
+void
+extract_header(std::vector<Tp>& _arr, rocprofiler_record_header_t* _hdr)
+{
+    if(_hdr->kind == typeid(Tp).hash_code())
+    {
+        auto* _v = reinterpret_cast<Tp*>(_hdr->payload);
+        _arr.emplace_back(*_v);
+    }
+    else
+    {
+        GTEST_FAIL() << __PRETTY_FUNCTION__ << " failed";
+    }
+}
+}  // namespace
+
+TEST(buffering, serial)
+{
+    // this test verifies that the buffering system is ordered properly
+    // and does not suffer from data loss or data corruption. We generate
+    // 240 raw arrays of data where 120 of them are twice as large as the
+    // the other 120 raw array and these two arrays contain data of different
+    // types. For each iteration, we randomize whether the uint64_t array with
+    // 32 elements or whether the double array with 64 elements gets inserted
+    // first. We then pull all the data back out of the buffer and verify
+    // that no arrays were lost and that none of the data was corrupted.
+
+    uint64_t n = 120;
+
+    // storage of the original data put into the buffer
+    auto _ui_history = std::vector<uint_raw_array_t>{};
+    auto _ui_result  = std::vector<uint_raw_array_t>{};
+
+    // storage of the data extracted from the buffer
+    auto _fp_history = std::vector<flt_raw_array_t>{};
+    auto _fp_result  = std::vector<flt_raw_array_t>{};
+
+    // a buffer to hold all the data
+    auto _buffer = record_header_buffer_t{n * (sizeof(uint_raw_array_t) + sizeof(flt_raw_array_t))};
+
+    // RNG use to make the ordering of the different sized records inconsistent
+    auto _gen = std::mt19937_64{std::random_device{}()};
+    auto _rng = std::uniform_int_distribution<short>{0, 1};
+    for(uint64_t i = 0; i < n; ++i)
+    {
+        // generate a 32*8 byte array
+        auto _u = generate_array<uint64_t, 32>();
+        // generate a 64*8 byte array
+        auto _f = generate_array<double, 64>();
+
+        // store the original data
+        _ui_history.emplace_back(_u);
+        _fp_history.emplace_back(_f);
+
+        EXPECT_EQ(_u, _ui_history.back()) << "uint not equal after emplace_back";
+        EXPECT_EQ(_f, _fp_history.back()) << "float not equal after emplace_back";
+
+        // randomize sequence of insertion into buffer
+        if(_rng(_gen) % 2 == 0)
+        {
+            _buffer.emplace(_u);
+            _buffer.emplace(_f);
+        }
+        else
+        {
+            _buffer.emplace(_f);
+            _buffer.emplace(_u);
+        }
+
+        EXPECT_EQ(_u, _ui_history.back()) << "uint not equal after emplace_back";
+        EXPECT_EQ(_f, _fp_history.back()) << "float not equal after emplace_back";
+    }
+
+    // get the records out of the buffer
+    auto _headers = _buffer.get_record_headers();
+    for(auto* itr : _headers)
+    {
+        ASSERT_TRUE(itr->payload) << "nullptr to payload not expected";
+
+        if(itr->kind == typeid(uint_raw_array_t).hash_code())
+        {
+            extract_header(_ui_result, itr);
+        }
+        else if(itr->kind == typeid(flt_raw_array_t).hash_code())
+        {
+            extract_header(_fp_result, itr);
+        }
+        else
+        {
+            GTEST_FAIL() << "unknown type id hash code: " << std::to_string(itr->kind);
+        }
+    }
+
+    // validate that we got the same number of records out that we put in
+    ASSERT_EQ(_ui_history.size(), _ui_result.size())
+        << "UINT: " << _ui_history.size() << " vs. " << _ui_result.size();
+    ASSERT_EQ(_fp_history.size(), _fp_result.size())
+        << "FLOAT: " << _fp_history.size() << " vs. " << _fp_result.size();
+
+    // validate there was no data corruption or data loss from storage in the buffer
+    for(size_t i = 0; i < n; ++i)
+    {
+        auto& _ui_lhs = _ui_history.at(i);
+        auto& _ui_rhs = _ui_result.at(i);
+        auto& _fp_lhs = _fp_history.at(i);
+        auto& _fp_rhs = _fp_result.at(i);
+
+        EXPECT_EQ(_ui_lhs, _ui_rhs) << "\n"
+                                    << "UINT LHS:\n"
+                                    << _ui_lhs.to_string() << "\n"
+                                    << "UINT RHS:\n"
+                                    << _ui_rhs.to_string() << "\n";
+
+        EXPECT_EQ(_fp_lhs, _fp_rhs) << "\n"
+                                    << "FLOAT LHS:\n"
+                                    << _fp_lhs.to_string() << "\n"
+                                    << "FLOAT RHS:\n"
+                                    << _fp_rhs.to_string() << "\n";
+    }
+}
@@ -0,0 +1,105 @@
+// MIT License
+//
+// Copyright (c) 2023 ROCm Developer Tools
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include "lib/common/container/record_header_buffer.hpp"
+
+#include <random>
+#include <string>
+#include <type_traits>
+
+namespace rocprofiler
+{
+namespace test
+{
+template <typename Tp, size_t N>
+struct raw_array
+{
+    raw_array()  = default;
+    ~raw_array() = default;
+
+    raw_array(const raw_array&)     = default;
+    raw_array(raw_array&&) noexcept = default;
+
+    raw_array& operator=(const raw_array&) = default;
+    raw_array& operator=(raw_array&&) noexcept = default;
+
+    bool operator==(const raw_array<Tp, N>& rhs) const;
+    bool operator!=(const raw_array<Tp, N>& rhs) const { return !(*this == rhs); }
+
+    Tp& operator[](size_t n) { return data[n]; }
+    Tp  operator[](size_t n) const { return data[n]; }
+
+    std::string to_string() const;
+
+    Tp data[N];
+};
+
+template <typename Tp, size_t N>
+bool
+raw_array<Tp, N>::operator==(const raw_array<Tp, N>& rhs) const
+{
+    for(size_t i = 0; i < N; ++i)
+    {
+        if constexpr(std::is_integral_v<Tp>)
+        {
+            if((*this)[i] != rhs[i]) return false;
+        }
+        else
+        {
+            auto _diff = (*this)[i] - rhs[i];
+            if(_diff < Tp{0.0}) _diff *= Tp{-1.0};
+            if(_diff > std::numeric_limits<Tp>::round_error()) return false;
+        }
+    }
+    return true;
+}
+
+template <typename Tp, size_t N>
+std::string
+raw_array<Tp, N>::to_string() const
+{
+    auto _ss = std::stringstream{};
+    for(size_t i = 0; i < N; ++i)
+    {
+        _ss << " " << std::setw(8) << std::fixed << std::setprecision(3) << (*this)[i];
+        if(i % 16 == 15) _ss << "\n";
+    }
+    return _ss.str();
+}
+
+template <typename Tp, size_t N>
+auto
+generate(raw_array<Tp, N>& _v, Tp _min, Tp _max)
+{
+    using rng_t = std::conditional_t<std::is_integral<Tp>::value,
+                                     std::uniform_int_distribution<Tp>,
+                                     std::uniform_real_distribution<Tp>>;
+    auto _rd    = std::random_device{};
+    auto _gen   = std::mt19937_64{_rd()};
+    auto _rng   = rng_t{_min, _max};
+    for(size_t i = 0; i < N; ++i)
+        _v[i] = _rng(_gen);
+}
+}  // namespace test
+}  // namespace rocprofiler