[SDK] Fix double buffer data race (#394)

* Fix double buffer data race

- fixes relatively rare data race in double buffering scheme

In `rocprofiler::buffer::instance::emplace`, the `container::record_header_buffer::get_record_headers()` function returned a `std::vector<rocprofiler_record_header_t*>` and then invoked callback to tool. It was possible for that callback to still be executing while the buffer was being updated. This potentially introduced a scenario where the rocprofiler_record_header_t* was modified (or corrupted) before the tool processed the record. In rocprofv3, this would result in a "future" buffer record showing up among "past" buffer records. E.g., correlation id sequence of 1-15 where the buffer flushes after five values, could result in this during processing:
|     |     |     |     |      |
|:---:|:---:|:---:|:---:|:---:|
|  1 |  2 |  3 |  4 | 15 |
|  6 |  7 |  8 |  9 | 10 |
| 11 | 12 | 13 | 14 | 15 |

Because buffer A (of double buffering scheme) originally containing corr ids 1-5 stalled after process corr id 4 (e.g. write to disk), buffer B filled up with 6-10 and started flushing, causing a switch back to buffer A, and buffer A was filled with 11-15 by the time callback accessed what was originally corr id 5 but was now updated to corr id 15.

* Update CHANGELOG

* misc minor cleanup

---------

Co-authored-by: Jonathan R. Madsen <jonathanrmadsen@gmail.com>

source/lib/common/container/record_header_buffer.cpp

@@ -82,20 +82,18 @@ record_header_buffer::allocate(size_t num_bytes)
     return true;
 }
 
-record_header_buffer::record_ptr_vec_t
-record_header_buffer::get_record_headers(size_t _n)
+size_t
+record_header_buffer::get_num_record_headers()
 {
     auto _lk = rhb_raii_lock{*this};
 
-    auto _sz  = m_index.load(std::memory_order_acquire);
-    _n        = std::min(_n, _sz);
-    auto _ret = record_ptr_vec_t{};
-    _ret.reserve(_n);
-    for(size_t i = 0; i < _n; ++i)
+    auto   _size = m_index.load(std::memory_order_acquire);
+    size_t _ret  = 0;
+    for(size_t i = 0; i < _size; ++i)
     {
-        if(auto& itr = m_headers.at(i); itr.hash > 0 && itr.payload != nullptr)
-            _ret.emplace_back(&itr);
+        if(auto& itr = m_headers.at(i); itr.hash > 0 && itr.payload != nullptr) ++_ret;
     }
+
     return _ret;
 }
 

source/lib/common/container/record_header_buffer.hpp

@@ -23,13 +23,16 @@
 #pragma once
 
 #include "lib/common/container/ring_buffer.hpp"
+#include "lib/common/scope_destructor.hpp"
 
 #include <rocprofiler-sdk/fwd.h>
 
 #include <atomic>
+#include <functional>
 #include <limits>
 #include <mutex>
 #include <shared_mutex>
+#include <type_traits>
 #include <vector>
 
 namespace rocprofiler
@@ -75,9 +78,13 @@ struct record_header_buffer
     template <typename Tp>
     bool emplace(uint32_t, uint32_t, Tp&);
 
-    /// this function will return a vector of pointers to the record headers
-    /// at the time of invocation.
-    record_ptr_vec_t get_record_headers(size_t _n = std::numeric_limits<size_t>::max());
+    /// this function will return the number of record headers
+    size_t get_num_record_headers();
+
+    /// this function will invoke functor with a vector of pointers to the record headers.
+    /// if ClearRecordsV is true, the container will be cleared after invoking the functor
+    template <typename ClearRecordsT, typename FuncT, typename... Args>
+    size_t process_record_headers(ClearRecordsT, FuncT&& functor, Args&&... args);
 
     /// record_header_buffer is a multiple writer, single reader data structure so
     /// this function prevents writing via emplace
@@ -322,6 +329,34 @@ record_header_buffer::emplace(Tp& _v)
     // if enumerations are not used, use the typeid hash code
     return emplace(typeid(Tp).hash_code(), _v);
 }
+
+template <typename ClearRecordsT, typename FuncT, typename... Args>
+size_t
+record_header_buffer::process_record_headers(ClearRecordsT, FuncT&& _functor, Args&&... _args)
+{
+    // RAII for lock/unlock
+    auto _lk = scope_destructor{[&]() { unlock(); }, [&]() { lock(); }};
+
+    auto _n       = m_index.load(std::memory_order_acquire);
+    auto _records = record_ptr_vec_t{};
+    _records.reserve(_n);
+    for(size_t i = 0; i < _n; ++i)
+    {
+        if(auto& itr = m_headers.at(i); itr.hash > 0 && itr.payload != nullptr)
+            _records.emplace_back(&itr);
+    }
+
+    // get number of records before vector is moved
+    auto _num_records = _records.size();
+
+    // invoke the callback
+    std::forward<FuncT>(_functor)(std::move(_records), std::forward<Args>(_args)...);
+
+    // clear the container
+    if constexpr(ClearRecordsT::value) clear();
+
+    return _num_records;
+}
 }  // namespace container
 }  // namespace common
 }  // namespace rocprofiler