[rocprofv3] MultiKernelDispatch ATT support (#774)

* Initial consecutive kernel WIP * Updated logic after discussion, create context only when needed, change set of captured ids to dispatch_id_t type * Updated to fix concurrency issues and revert kernel_iterations * Add captured id in first lock capture * Updated code to use wlock, added comments, removed some unecessary atomic * Cleaned up, need to add test * Add test to check that generated stats csv file is not empty * Updated test to check if vector-ops kernels are being used * Fix phase bug * Updated for comments * Flattened ATT logic a bit * Fix incorrect if-statement * Fix merge conflict
2025-09-23 20:19:27 -05:00
parent 34b9184686
commit abd6029603
8 changed files with 398 additions and 29 deletions
@@ -794,6 +794,13 @@ For attachment profiling of running processes:
        type=str,
    )

+    att_options.add_argument(
+        "--att-consecutive-kernels",
+        help="Consecutive kernels to profile with ATT. Default 0",
+        default=None,
+        type=str,
+    )
+
    att_options.add_argument(
        "--att-buffer-size",
        help="Thread trace buffer size. Default 256MB",
@@ -1627,6 +1634,12 @@ def run(app_args, args, **kwargs):
                int_auto(args.att_simd_select),
                overwrite=True,
            )
+        if args.att_consecutive_kernels:
+            update_env(
+                "ROCPROF_ATT_CONSECUTIVE_KERNELS",
+                int_auto(args.att_consecutive_kernels),
+                overwrite=True,
+            )
        if args.att_serialize_all:
            update_env(
                "ROCPROF_ATT_PARAM_SERIALIZE_ALL",
@@ -151,6 +151,7 @@ struct config : output_config
    uint64_t att_param_simd_select = get_env<uint64_t>("ROCPROF_ATT_PARAM_SIMD_SELECT", 0xF);
    uint64_t att_param_target_cu   = get_env<uint64_t>("ROCPROF_ATT_PARAM_TARGET_CU", 1);
    uint64_t att_param_perf_ctrl   = get_env<uint64_t>("ROCPROF_ATT_PARAM_PERFCOUNTER_CTRL", 0);
+    uint64_t att_consecutive_kernels = get_env<uint64_t>("ROCPROF_ATT_CONSECUTIVE_KERNELS", 0);

    std::string kernel_filter_include   = get_env("ROCPROF_KERNEL_FILTER_INCLUDE_REGEX", ".*");
    std::string kernel_filter_exclude   = get_env("ROCPROF_KERNEL_FILTER_EXCLUDE_REGEX", "");
@@ -303,6 +304,7 @@ config::save(ArchiveT& ar) const
    CFG_SERIALIZE_MEMBER(att_library_path);
    CFG_SERIALIZE_MEMBER(att_param_perfcounters);
    CFG_SERIALIZE_MEMBER(att_param_perf_ctrl);
+    CFG_SERIALIZE_MEMBER(att_consecutive_kernels);

    // serialize the base class
    static_cast<const base_type*>(this)->save(ar);
@@ -259,10 +259,13 @@ using kernel_iteration_t    = std::unordered_map<rocprofiler_kernel_id_t, size_t
 using kernel_rename_map_t   = std::unordered_map<uint64_t, uint64_t>;
 using kernel_rename_stack_t = std::stack<uint64_t>;

-auto*      tool_metadata     = as_pointer<tool::metadata>(tool::metadata::inprocess{});
-auto       target_kernels    = common::Synchronized<targeted_kernels_map_t>{};
-auto*      execution_profile = as_pointer<common::Synchronized<tool::execution_profile_data>>();
-auto       counter_collection_ctx = rocprofiler_context_id_t{0};
+auto* tool_metadata          = as_pointer<tool::metadata>(tool::metadata::inprocess{});
+auto  target_kernels         = common::Synchronized<targeted_kernels_map_t>{};
+auto* execution_profile      = as_pointer<common::Synchronized<tool::execution_profile_data>>();
+auto  counter_collection_ctx = rocprofiler_context_id_t{0};
+auto  att_device_context     = rocprofiler_context_id_t{0};
+auto  att_consecutive_kernel_dispatch_id =
+    std::atomic<rocprofiler_dispatch_id_t>{std::numeric_limits<uint64_t>::max()};
 std::mutex att_shader_data;

 thread_local auto thread_dispatch_rename      = as_pointer<kernel_rename_stack_t>();
@@ -1390,9 +1393,12 @@ att_shader_data_callback(rocprofiler_agent_id_t  agent,
 {
    std::lock_guard<std::mutex> lock(att_shader_data);
    std::stringstream           filename;
-    filename << fmt::format("{}_shader_engine_{}_{}", agent.handle, se_id, userdata.value);
+    auto dispatch_id = static_cast<rocprofiler_dispatch_id_t>(userdata.value);
+    // If dispatch_id/userdata.value == 0, then we are in device mode and get dispatch id from
+    // global atomic
+    if(dispatch_id == 0) dispatch_id = att_consecutive_kernel_dispatch_id.load();
+    filename << fmt::format("{}_shader_engine_{}_{}", agent.handle, se_id, dispatch_id);

-    auto        dispatch_id     = static_cast<rocprofiler_dispatch_id_t>(userdata.value);
    auto        output_stream   = get_output_stream(tool::get_config(), filename.str(), ".att");
    std::string output_filename = get_output_filename(tool::get_config(), filename.str(), ".att");

@@ -1411,14 +1417,97 @@ att_dispatch_callback(rocprofiler_agent_id_t /* agent_id  */,
                      rocprofiler_user_data_t* userdata_shader)
 {
    static auto kernel_iteration = common::Synchronized<kernel_iteration_t, true>{};
-
-    userdata_shader->value = dispatch_id;
+    userdata_shader->value       = dispatch_id;

    if(is_targeted_kernel(kernel_id, kernel_iteration))
        return ROCPROFILER_THREAD_TRACE_CONTROL_START_AND_STOP;
    return ROCPROFILER_THREAD_TRACE_CONTROL_NONE;
 }

+void
+att_dispatch_consecutive_kernel_callback(rocprofiler_callback_tracing_record_t record,
+                                         rocprofiler_user_data_t* /*user_data*/,
+                                         void* userdata)
+{
+    using capture_ids_set_t = common::Synchronized<std::unordered_set<rocprofiler_dispatch_id_t>>;
+    if(record.kind != ROCPROFILER_CALLBACK_TRACING_KERNEL_DISPATCH) return;
+    if(record.phase == ROCPROFILER_CALLBACK_PHASE_EXIT) return;
+
+    ROCP_FATAL_IF(record.payload == nullptr)
+        << fmt::format("Expected record payload to not be null for {}", __FUNCTION__);
+    static auto kernel_iteration = common::Synchronized<kernel_iteration_t, true>{};
+    auto* rdata = static_cast<rocprofiler_callback_tracing_kernel_dispatch_data_t*>(record.payload);
+    auto  dispatch_id = rdata->dispatch_info.dispatch_id;
+    auto  kernel_id   = rdata->dispatch_info.kernel_id;
+
+    // Keep track of number of consecutive kernels
+    const auto consecutive_kernels = *static_cast<uint64_t*>(CHECK_NOTNULL(userdata));
+
+    static std::atomic<bool> isprofiling{false};
+    static bool              stop_profiling{false};
+    static size_t            num_consecutive_kernels{0};
+    static capture_ids_set_t captured_ids{};
+
+    if(record.phase == ROCPROFILER_CALLBACK_PHASE_ENTER)
+    {
+        const auto is_target = is_targeted_kernel(kernel_id, kernel_iteration);
+        // Return if kernel is not targeted and we are not profiling currently
+        if(!is_target && !isprofiling.load()) return;
+
+        captured_ids.wlock(
+            [](std::unordered_set<rocprofiler_dispatch_id_t>& _data,
+               const rocprofiler_dispatch_id_t                _dispatch_id,
+               const bool                                     _is_target,
+               const uint64_t                                 _consecutive_kernels) {
+                // Reset consecutive kernel count and start context if not already started
+                if(_is_target) num_consecutive_kernels = 0;
+                // Start context if target and not started already
+                if(_is_target && !isprofiling.load())
+                {
+                    ROCPROFILER_CALL(rocprofiler_start_context(att_device_context),
+                                     "context start");
+                    isprofiling.store(true);
+                }
+                const auto local_count = num_consecutive_kernels++;
+                if(isprofiling && local_count < _consecutive_kernels)
+                {
+                    // Keep track of launched dispatch ids
+                    _data.emplace(_dispatch_id);
+                    // Store lowest dispatch id for shader callback function
+                    if(att_consecutive_kernel_dispatch_id.load() > _dispatch_id)
+                        att_consecutive_kernel_dispatch_id.store(_dispatch_id);
+                }
+                if(local_count >= _consecutive_kernels) stop_profiling = true;
+            },
+            dispatch_id,
+            is_target,
+            consecutive_kernels);
+
+        return;
+    }
+
+    ROCP_CI_LOG_IF(WARNING, record.phase != ROCPROFILER_CALLBACK_PHASE_NONE) << fmt::format(
+        "Expected record phase to be ROCPROFILER_CALLBACK_PHASE_NONE for {}", __FUNCTION__);
+
+    if(!isprofiling) return;
+
+    // Stop profiling if all captured dispatches have finished
+    captured_ids.wlock(
+        [](std::unordered_set<rocprofiler_dispatch_id_t>& _data,
+           rocprofiler_dispatch_id_t                      _dispatch_id) {
+            _data.erase(_dispatch_id);
+            if(!_data.empty() || !stop_profiling) return;
+
+            bool _exp = true;
+            if(!isprofiling.compare_exchange_strong(_exp, false, std::memory_order_relaxed)) return;
+
+            ROCPROFILER_CALL(rocprofiler_stop_context(att_device_context), "context stop");
+            stop_profiling = false;
+            att_consecutive_kernel_dispatch_id.store(std::numeric_limits<uint64_t>::max());
+        },
+        dispatch_id);
+}
+
 void
 counter_dispatch_callback(rocprofiler_dispatch_counting_service_data_t dispatch_data,
                          rocprofiler_counter_config_id_t*             config,
@@ -1726,6 +1815,7 @@ struct tracing_callbacks_t
    , att_shader_data{att_shader_data_callback}
    , counter_dispatch{counter_dispatch_callback}
    , counter_record{counter_record_callback}
+    , att_dispatch_consecutive_kernel{att_dispatch_consecutive_kernel_callback}
    {}

    explicit tracing_callbacks_t(dummy_callbacks_t)
@@ -1740,17 +1830,18 @@ struct tracing_callbacks_t
    , counter_record{dummy_counter_record_callback}
    {}

-    const rocprofiler_callback_tracing_cb_t               code_object_tracing = nullptr;
-    const rocprofiler_callback_tracing_cb_t               cntrl_tracing       = nullptr;
-    const rocprofiler_callback_tracing_cb_t               kernel_rename       = nullptr;
-    const rocprofiler_callback_tracing_cb_t               hip_stream          = nullptr;
-    const rocprofiler_callback_tracing_cb_t               callback_tracing    = nullptr;
-    const rocprofiler_buffer_tracing_cb_t                 buffered_tracing    = nullptr;
-    const rocprofiler_buffer_tracing_cb_t                 pc_sampling         = nullptr;
-    const rocprofiler_thread_trace_dispatch_callback_t    att_dispatch        = nullptr;
-    const rocprofiler_thread_trace_shader_data_callback_t att_shader_data     = nullptr;
-    const rocprofiler_dispatch_counting_service_cb_t      counter_dispatch    = nullptr;
-    const rocprofiler_dispatch_counting_record_cb_t       counter_record      = nullptr;
+    const rocprofiler_callback_tracing_cb_t               code_object_tracing             = nullptr;
+    const rocprofiler_callback_tracing_cb_t               cntrl_tracing                   = nullptr;
+    const rocprofiler_callback_tracing_cb_t               kernel_rename                   = nullptr;
+    const rocprofiler_callback_tracing_cb_t               hip_stream                      = nullptr;
+    const rocprofiler_callback_tracing_cb_t               callback_tracing                = nullptr;
+    const rocprofiler_buffer_tracing_cb_t                 buffered_tracing                = nullptr;
+    const rocprofiler_buffer_tracing_cb_t                 pc_sampling                     = nullptr;
+    const rocprofiler_thread_trace_dispatch_callback_t    att_dispatch                    = nullptr;
+    const rocprofiler_thread_trace_shader_data_callback_t att_shader_data                 = nullptr;
+    const rocprofiler_dispatch_counting_service_cb_t      counter_dispatch                = nullptr;
+    const rocprofiler_dispatch_counting_record_cb_t       counter_record                  = nullptr;
+    const rocprofiler_callback_tracing_cb_t               att_dispatch_consecutive_kernel = nullptr;
 };

 auto
@@ -2078,6 +2169,25 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)

        const auto selecting_by_gpuid = !gpu_idx_set.empty();

+        // Use device_thread_trace_service when handling consecutive kernels
+        const auto handle_consecutive_kernels = tool::get_config().att_consecutive_kernels >= 1;
+        rocprofiler_user_data_t user{.value = 0};
+
+        if(handle_consecutive_kernels)
+        {
+            // Use user data pointer to dispatch id to communicate dispatch ID to shader callback
+            // function
+            ROCPROFILER_CALL(rocprofiler_create_context(&att_device_context), "context creation");
+            ROCPROFILER_CALL(rocprofiler_configure_callback_tracing_service(
+                                 get_client_ctx(),
+                                 ROCPROFILER_CALLBACK_TRACING_KERNEL_DISPATCH,
+                                 nullptr,
+                                 0,
+                                 callbacks.att_dispatch_consecutive_kernel,
+                                 static_cast<void*>(&tool::get_config().att_consecutive_kernels)),
+                             "dispatch tracing service configure");
+        }
+
        for(auto& [id, agent] : tool_metadata->agents_map)
        {
            if(agent.type != ROCPROFILER_AGENT_TYPE_GPU) continue;
@@ -2087,16 +2197,29 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
            auto agent_params = global_parameters;
            for(auto& counter : get_att_perfcounter_params(id, att_perf))
                agent_params.push_back(counter);
-
-            ROCPROFILER_CALL(
-                rocprofiler_configure_dispatch_thread_trace_service(get_client_ctx(),
-                                                                    id,
-                                                                    agent_params.data(),
-                                                                    agent_params.size(),
-                                                                    callbacks.att_dispatch,
-                                                                    callbacks.att_shader_data,
-                                                                    tool_data),
-                "thread trace service configure");
+            if(!handle_consecutive_kernels)
+            {
+                ROCPROFILER_CALL(
+                    rocprofiler_configure_dispatch_thread_trace_service(get_client_ctx(),
+                                                                        id,
+                                                                        agent_params.data(),
+                                                                        agent_params.size(),
+                                                                        callbacks.att_dispatch,
+                                                                        callbacks.att_shader_data,
+                                                                        tool_data),
+                    "thread trace service configure");
+            }
+            else
+            {
+                ROCPROFILER_CALL(
+                    rocprofiler_configure_device_thread_trace_service(att_device_context,
+                                                                      agent.id,
+                                                                      agent_params.data(),
+                                                                      agent_params.size(),
+                                                                      callbacks.att_shader_data,
+                                                                      user),
+                    "thread trace service configure");
+            }
        }

        // Any agent not removed by above loop was not in the agents_map list
@@ -50,3 +50,4 @@ add_subdirectory(rocpd)
 add_subdirectory(rocpd-kernel-rename)
 add_subdirectory(attachment)
 add_subdirectory(rocpd-scratch)
+add_subdirectory(att-consecutive-kernels)
@@ -0,0 +1,91 @@
+# MIT License
+#
+# Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+#
+# rocprofv3 tool test
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+include(FindPackageHandleStandardArgs)
+
+project(
+    rocprofiler-sdk-tests-rocprofv3-att
+    LANGUAGES CXX
+    VERSION 0.0.0)
+
+set(CMAKE_MESSAGE_INDENT "[${PROJECT_NAME}] ")
+
+string(REPLACE "LD_PRELOAD=" "ROCPROF_PRELOAD=" PRELOAD_ENV
+               "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}")
+
+rocprofiler_configure_pytest_files(CONFIG pytest.ini COPY validate.py conftest.py)
+
+find_package(rocprofiler-sdk REQUIRED)
+
+find_library(
+    attdecoder_LIBRARY
+    NAMES rocprof-trace-decoder
+    HINTS ${ROCM_PATH}
+    PATHS ${ROCM_PATH}
+    PATH_SUFFIXES lib)
+
+if(attdecoder_LIBRARY)
+    cmake_path(GET attdecoder_LIBRARY PARENT_PATH attdecoder_LIB_DIR)
+endif()
+
+find_package_handle_standard_args(attdecoder REQUIRED_VARS attdecoder_LIB_DIR
+                                                           attdecoder_LIBRARY)
+
+set(IS_DISABLED ON)
+if(attdecoder_FOUND)
+    set(IS_DISABLED OFF)
+    set(ATT_LIB --att-library-path ${attdecoder_LIB_DIR})
+endif()
+
+# consecutive kernel test
+add_test(
+    NAME rocprofv3-test-att-consecutive-kernels-execute
+    COMMAND
+        $<TARGET_FILE:rocprofiler-sdk::rocprofv3> ${ATT_LIB} --att
+        --att-consecutive-kernels 8 -d ${CMAKE_CURRENT_BINARY_DIR}/%tag%-trace -o out
+        --log-level env -- $<TARGET_FILE:vector-ops>)
+
+set_tests_properties(
+    rocprofv3-test-att-consecutive-kernels-execute
+    PROPERTIES TIMEOUT 45 LABELS "integration-tests" DISABLED ${IS_DISABLED})
+
+add_test(NAME rocprofv3-test-att-consecutive-kernels-validate
+         COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py
+                 --csv-directory-input ${CMAKE_CURRENT_BINARY_DIR}/vector-ops-trace/)
+
+set_tests_properties(
+    rocprofv3-test-att-consecutive-kernels-validate
+    PROPERTIES TIMEOUT
+               60
+               LABELS
+               "integration-tests"
+               DEPENDS
+               rocprofv3-test-att-consecutive-kernels-execute
+               FAIL_REGULAR_EXPRESSION
+               "AssertionError"
+               DISABLED
+               ${IS_DISABLED})
@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+# MIT License
+#
+# Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import csv
+import pytest
+import json
+
+from rocprofiler_sdk.pytest_utils.dotdict import dotdict
+from rocprofiler_sdk.pytest_utils import collapse_dict_list
+from rocprofiler_sdk.pytest_utils.perfetto_reader import PerfettoReader
+
+import re
+import os
+
+
+def pytest_addoption(parser):
+
+    parser.addoption(
+        "--csv-directory-input",
+        action="store",
+        help="Path to stats csv file.",
+    )
+
+
+@pytest.fixture
+def csv_data(request):
+    import glob
+
+    directory = request.config.getoption("--csv-directory-input")
+
+    data = []
+    if not os.path.isdir(directory):
+        raise FileExistsError(f"{directory} does not exist")
+
+    stats_glob = os.path.join(directory, "stats*.csv")
+    for filename in glob.glob(stats_glob):
+        file_data = []
+        with open(filename, "r") as inp:
+            reader = csv.DictReader(inp)
+            for row in reader:
+                file_data.append(row)
+        data.append(file_data)
+    return data
@@ -0,0 +1,27 @@
+# MIT License
+#
+# Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+
+[pytest]
+addopts = --durations=20 -rA -s -vv
+testpaths = validate.py
+pythonpath = @ROCPROFILER_SDK_TESTS_BINARY_DIR@/pytest-packages
@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+# MIT License
+#
+# Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import re
+import sys
+import pytest
+
+
+def test_csv_data(csv_data):
+    assert len(csv_data) > 0, "Expected at least one stats csv file"
+
+    kernel_pattern = re.compile("kernel")
+    MIN_EXPECTED_LINES = 5
+    kernel_set = set()
+    for stats_csv_data in csv_data:
+        assert (
+            len(stats_csv_data) >= MIN_EXPECTED_LINES
+        ), "Expected stats csv file to be non-empty"
+
+        for row in stats_csv_data:
+            if kernel_pattern.search(row["Source"]):
+                kernel_set.add(row["Source"])
+
+    assert len(kernel_set) >= 4, "Expected all vector-ops kernels to be recorded"
+
+
+if __name__ == "__main__":
+    exit_code = pytest.main(["-x", __file__] + sys.argv[1:])
+    sys.exit(exit_code)