[SDK] Fix double buffer data race (#394)

* Fix double buffer data race

- fixes relatively rare data race in double buffering scheme

In `rocprofiler::buffer::instance::emplace`, the `container::record_header_buffer::get_record_headers()` function returned a `std::vector<rocprofiler_record_header_t*>` and then invoked callback to tool. It was possible for that callback to still be executing while the buffer was being updated. This potentially introduced a scenario where the rocprofiler_record_header_t* was modified (or corrupted) before the tool processed the record. In rocprofv3, this would result in a "future" buffer record showing up among "past" buffer records. E.g., correlation id sequence of 1-15 where the buffer flushes after five values, could result in this during processing:
|     |     |     |     |      |
|:---:|:---:|:---:|:---:|:---:|
|  1 |  2 |  3 |  4 | 15 |
|  6 |  7 |  8 |  9 | 10 |
| 11 | 12 | 13 | 14 | 15 |

Because buffer A (of double buffering scheme) originally containing corr ids 1-5 stalled after process corr id 4 (e.g. write to disk), buffer B filled up with 6-10 and started flushing, causing a switch back to buffer A, and buffer A was filled with 11-15 by the time callback accessed what was originally corr id 5 but was now updated to corr id 15.

* Update CHANGELOG

* misc minor cleanup

---------

Co-authored-by: Jonathan R. Madsen <jonathanrmadsen@gmail.com>

source/lib/tests/buffering/buffering-parallel.cpp

@@ -212,6 +212,11 @@ TEST(buffering, parallel)
     // wait for all the threads to complete
     pthread_barrier_wait(&_emplaced_barrier);
 
+    // designates that buffer should be cleared after invoking functor
+    using clear_buffer_t = std::true_type;
+
     // verify the data pulled out the buffer matches the data put in by the threads
-    validate(_buffer.get_record_headers(), test_data_types{}, test_data_sizes);
+    _buffer.process_record_headers(clear_buffer_t{}, [](auto&& _records) {
+        validate(_records, test_data_types{}, test_data_sizes);
+    });
 }

source/lib/tests/buffering/buffering-save-load.cpp

@@ -180,8 +180,10 @@ TEST(buffering, save_load)
     // and move it to another object and ensure that the data after the save + load + move matches
     // the original data placed into the buffer without any data corruption or loss
 
-    constexpr auto num_variants = test_data_types::size() * test_data_sizes.size();
-    constexpr auto data_size    = get_data_size(test_data_types{}, test_data_sizes);
+    // designates that buffer should not be cleared after invoking functor
+    constexpr auto clear_buffer_v = std::false_type{};
+    constexpr auto num_variants   = test_data_types::size() * test_data_sizes.size();
+    constexpr auto data_size      = get_data_size(test_data_types{}, test_data_sizes);
 
     EXPECT_EQ(num_variants, 120);
 
@@ -224,7 +226,9 @@ TEST(buffering, save_load)
     EXPECT_FALSE(_buffer.is_empty());
 
     // verify the data pulled out the buffer matches the data put in
-    validate(_buffer.get_record_headers(), test_data_types{}, test_data_sizes);
+    _buffer.process_record_headers(clear_buffer_v, [](auto&& _records) {
+        validate(_records, test_data_types{}, test_data_sizes);
+    });
 
     // save the data to a binary file and clear the buffer so it can "receive" new data (in theory)
     {
@@ -235,7 +239,7 @@ TEST(buffering, save_load)
     }
 
     // verify that the buffer is empty
-    EXPECT_EQ(_buffer.get_record_headers().size(), 0) << "buffer was not cleared properly";
+    EXPECT_EQ(_buffer.get_num_record_headers(), 0) << "buffer was not cleared properly";
 
     // load the data back from the binary file
     {
@@ -245,23 +249,26 @@ TEST(buffering, save_load)
     }
 
     // verify that, at a high level, all the data was preserved
-    ASSERT_EQ(_buffer.get_record_headers().size(), num_variants)
+    ASSERT_EQ(_buffer.get_num_record_headers(), num_variants)
         << "buffer was not saved/loaded properly";
 
     // verify the data is entirely correct
-    validate(_buffer.get_record_headers(), test_data_types{}, test_data_sizes);
+    _buffer.process_record_headers(clear_buffer_v, [](auto&& _records) {
+        validate(_records, test_data_types{}, test_data_sizes);
+    });
 
     // move the data into another instance of record_header_buffer_t
     auto _buffer_v = record_header_buffer_t{std::move(_buffer)};
 
     // make sure the move emptied out the old object and populated the new object
-    ASSERT_EQ(_buffer.get_record_headers().size(), 0) << "buffer was not moved properly";
-    ASSERT_EQ(_buffer_v.get_record_headers().size(), num_variants)
-        << "buffer was not moved properly";
+    ASSERT_EQ(_buffer.get_num_record_headers(), 0) << "buffer was not moved properly";
+    ASSERT_EQ(_buffer_v.get_num_record_headers(), num_variants) << "buffer was not moved properly";
 
     // validate the data in the new object
     // verify the data pulled out the buffer matches the data put in by the threads
-    validate(_buffer_v.get_record_headers(), test_data_types{}, test_data_sizes);
+    _buffer.process_record_headers(clear_buffer_v, [](auto&& _records) {
+        validate(_records, test_data_types{}, test_data_sizes);
+    });
 
     // make sure reset works when empty and when full
     EXPECT_EQ(_buffer.reset(), 0) << "buffer should be empty after move";

source/lib/tests/buffering/buffering-serial.cpp

@@ -123,26 +123,37 @@ TEST(buffering, serial)
         EXPECT_EQ(_f, _fp_history.back()) << "float not equal after emplace_back";
     }
 
+    // designates that buffer should be cleared after invoking functor
+    constexpr auto clear_buffer_v = std::true_type{};
+
     // get the records out of the buffer
-    auto _headers = _buffer.get_record_headers();
-    for(auto* itr : _headers)
-    {
-        ASSERT_TRUE(itr->payload) << "nullptr to payload not expected";
+    auto _num_headers = _buffer.process_record_headers(
+        clear_buffer_v,
+        [](auto&& _headers, auto& _ui_result_v, auto& _fp_result_v) {
+            for(auto* itr : _headers)
+            {
+                ASSERT_TRUE(itr->payload) << "nullptr to payload not expected";
 
-        if(itr->hash == typeid(uint_raw_array_t).hash_code())
-        {
-            extract_header(_ui_result, itr);
-        }
-        else if(itr->hash == typeid(flt_raw_array_t).hash_code())
-        {
-            extract_header(_fp_result, itr);
-        }
-        else
-        {
-            GTEST_FAIL() << "unknown type id hash code: " << std::to_string(itr->hash);
-        }
-    }
+                if(itr->hash == typeid(uint_raw_array_t).hash_code())
+                {
+                    extract_header(_ui_result_v, itr);
+                }
+                else if(itr->hash == typeid(flt_raw_array_t).hash_code())
+                {
+                    extract_header(_fp_result_v, itr);
+                }
+                else
+                {
+                    GTEST_FAIL() << "unknown type id hash code: " << std::to_string(itr->hash);
+                }
+            }
+        },
+        _ui_result,
+        _fp_result);
 
+    ASSERT_EQ(_ui_history.size() + _fp_history.size(), _num_headers)
+        << "UINT: " << _ui_history.size() << " + FLOAT: " << _fp_history.size()
+        << " != HEADERS: " << _num_headers;
     // validate that we got the same number of records out that we put in
     ASSERT_EQ(_ui_history.size(), _ui_result.size())
         << "UINT: " << _ui_history.size() << " vs. " << _ui_result.size();