[rocprofiler-sdk] Fix hip compiler table initialization after finalization (#1174)

* [rocprofiler-sdk] Fix hip compiler table initialization after finalization

- Resolves tickets
  - https://ontrack-internal.amd.com/browse/SWDEV-557219
  - https://ontrack-internal.amd.com/browse/SWDEV-505503

* Tweak log message

* Remove unsupported hip limit enums

- hipLimitDevRuntimeSyncDepth
- hipLimitDevRuntimePendingLaunchCount

* Update conftest.py

Co-authored-by: Mark Meserve <mark.meserve@amd.com>

* Update README.md

Co-authored-by: Mark Meserve <mark.meserve@amd.com>

* Update hip_host.cpp

---------

Co-authored-by: Mark Meserve <mark.meserve@amd.com>

projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/registration.cpp

@@ -982,6 +982,19 @@ rocprofiler_set_api_table(const char* name,
     ROCP_INFO << __FUNCTION__ << "(\"" << name << "\", " << lib_version << ", " << lib_instance
               << ", ..., " << num_tables << ")";
 
+    // if finalized/finalizing, ignore
+    if(rocprofiler::registration::get_fini_status() != 0)
+    {
+        ROCP_WARNING << fmt::format(
+            R"(rocprofiler-sdk has been finalized, ignoring {}(name="{}", lib_version={}, lib_instance={}, ..., num_tables={}) ...)",
+            __FUNCTION__,
+            name,
+            lib_version,
+            lib_instance,
+            num_tables);
+        return 0;
+    }
+
     static auto _once = std::once_flag{};
     std::call_once(_once, rocprofiler::registration::initialize);
 

projects/rocprofiler-sdk/tests/CMakeLists.txt

@@ -82,6 +82,7 @@ add_subdirectory(counter-collection)
 add_subdirectory(openmp-tools)
 add_subdirectory(rocdecode)
 add_subdirectory(rocjpeg)
+add_subdirectory(hip-host-tracing)
 
 # rocpd validation tests
 add_subdirectory(rocpd)

projects/rocprofiler-sdk/tests/bin/CMakeLists.txt

@@ -41,3 +41,4 @@ add_subdirectory(hsa-code-object)
 add_subdirectory(hip-streams)
 add_subdirectory(hip-streams-per-thread)
 add_subdirectory(attachment-test)
+add_subdirectory(hip-host)

projects/rocprofiler-sdk/tests/bin/hip-host/CMakeLists.txt

@@ -0,0 +1,18 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+project(rocprofiler-sdk-tests-bin-hip-host LANGUAGES CXX)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+find_package(hip REQUIRED)
+find_package(Threads REQUIRED)
+
+add_executable(hip-host)
+target_sources(hip-host PRIVATE hip_host.cpp)
+target_link_libraries(hip-host PRIVATE rocprofiler-sdk::tests-build-flags hip::host
+                                       Threads::Threads)

projects/rocprofiler-sdk/tests/bin/hip-host/README.md

@@ -0,0 +1,12 @@
+# hip-host Test Executable
+
+This application makes various calls to the HIP runtime in an application without device code.
+Without any device code present, the HIP compiler should not generate any calls to
+`__hipRegisterFatBinary`, `__hipRegisterFunction`, etc. Thus, this application makes an explicit
+call to `__hipUnregisterFatBinary(nullptr)` in the destructor of a global variable --
+which (should) result in the destructor being invoked after rocprofiler-sdk has
+finalized. The intention is to trigger the initialization of the `HipCompilerDispatchTable`
+after rocprofiler-sdk has finalized. When a rocprofiler-sdk tool is loaded, the output should
+have the following message:
+
+> `... registration.cpp:###] rocprofiler-sdk has been finalized, ignoring rocprofiler_set_api_table("hip_compiler", 60400, 0, ..., 1) call`

projects/rocprofiler-sdk/tests/bin/hip-host/hip_host.cpp

@@ -0,0 +1,312 @@
+// MIT License
+//
+// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#include <hip/hip_runtime.h>
+
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iomanip>
+#include <iostream>
+#include <memory>
+#include <set>
+#include <string>
+#include <string_view>
+#include <vector>
+
+#define HIP_CHECK(call)                                                                            \
+    do                                                                                             \
+    {                                                                                              \
+        hipError_t _e = (call);                                                                    \
+        if(_e != hipSuccess)                                                                       \
+        {                                                                                          \
+            std::cerr << "HIP error " << hipGetErrorName(_e) << " (" << static_cast<int>(_e)       \
+                      << ") at " << __FILE__ << ":" << __LINE__ << " -> " << hipGetErrorString(_e) \
+                      << std::endl;                                                                \
+            std::exit(EXIT_FAILURE);                                                               \
+        }                                                                                          \
+    } while(0)
+
+extern "C" {
+extern void
+__hipUnregisterFatBinary(void** modules);
+}
+
+namespace
+{
+struct dtor
+{
+    dtor() = default;
+    ~dtor()
+    {
+        std::cerr << "\nCalling __hipUnregisterFatBinary(nullptr)... \n" << std::flush;
+        __hipUnregisterFatBinary(nullptr);
+        std::cerr << "Calling __hipUnregisterFatBinary(nullptr)... Done\n" << std::flush;
+    }
+};
+
+const char*
+funcCacheToStr(hipFuncCache_t v)
+{
+    switch(v)
+    {
+        case hipFuncCachePreferNone: return "PreferNone";
+        case hipFuncCachePreferShared: return "PreferShared";
+        case hipFuncCachePreferL1: return "PreferL1";
+        case hipFuncCachePreferEqual: return "PreferEqual";
+        default: return "Unknown";
+    }
+}
+
+const char*
+shmemBankToStr(hipSharedMemConfig v)
+{
+    switch(v)
+    {
+        case hipSharedMemBankSizeDefault: return "Default";
+        case hipSharedMemBankSizeFourByte: return "FourByte";
+        case hipSharedMemBankSizeEightByte: return "EightByte";
+        default: return "Unknown";
+    }
+}
+
+struct LimitDesc
+{
+    hipLimit_t  limit;
+    const char* name;
+};
+
+const std::vector<LimitDesc> kLimits = {
+    {hipLimitStackSize, "hipLimitStackSize"},
+    {hipLimitPrintfFifoSize, "hipLimitPrintfFifoSize"},
+    {hipLimitMallocHeapSize, "hipLimitMallocHeapSize"},
+};
+
+void
+printBytes(const char* label, size_t bytes, int width = 28)
+{
+    const char* units[] = {"B", "KiB", "MiB", "GiB", "TiB"};
+    double      v       = static_cast<double>(bytes);
+    int         u       = 0;
+    while(v >= 1024.0 && u < 4)
+    {
+        v /= 1024.0;
+        ++u;
+    }
+    std::cout << std::left << std::setw(width) << label << ": " << bytes << " (" << std::fixed
+              << std::setprecision(2) << v << " " << units[u] << ")\n";
+}
+
+void
+printArray3(const char* label, int v[3], int width = 28)
+{
+    std::cout << std::left << std::setw(width) << label << ": " << v[0] << ", " << v[1] << ", "
+              << v[2] << "\n";
+}
+
+void
+printBool(const char* label, int v, int width = 28)
+{
+    std::cout << std::left << std::setw(width) << label << ": " << (v ? "true" : "false") << "\n";
+}
+
+void
+printIfNonzero(const char* label, int v, int width = 28)
+{
+    if(v != 0) std::cout << std::left << std::setw(width) << label << ": " << v << "\n";
+}
+}  // namespace
+
+auto _dtor = std::unique_ptr<dtor>{};
+
+int
+main(int argc, char** argv)
+{
+    _dtor = std::make_unique<dtor>();
+
+    const auto help_flags     = std::set<std::string_view>{"-h", "--help", "-?"};
+    auto       report_devices = std::set<int>{};
+    for(int i = 1; i < argc; ++i)
+    {
+        if(std::string_view{argv[i]}.find_first_not_of("0123456789") == std::string_view::npos)
+        {
+            report_devices.insert(std::atoi(argv[++i]));
+        }
+        else if(help_flags.count(std::string_view{argv[i]}) != 0)
+        {
+            std::cout << "Usage: " << argv[0] << "-h / --help / -? / [<device_id> ...]\n";
+            return EXIT_SUCCESS;
+        }
+        else
+        {
+            std::cerr << "Usage: " << argv[0] << " [--skip-device N ...]\n";
+            return EXIT_FAILURE;
+        }
+    }
+
+    int        deviceCount = 0;
+    hipError_t e           = hipGetDeviceCount(&deviceCount);
+    if(e == hipErrorNoDevice)
+    {
+        std::cout << "No HIP devices found.\n";
+        return EXIT_FAILURE;
+    }
+    HIP_CHECK(e);
+
+    std::cout << "Found " << deviceCount << " HIP device(s)\n\n";
+    if(report_devices.empty())
+    {
+        for(int i = 0; i < deviceCount; ++i)
+            report_devices.insert(i);
+    }
+
+    for(int dev : report_devices)
+    {
+        HIP_CHECK(hipSetDevice(dev));
+
+        hipDeviceProp_t prop{};
+        HIP_CHECK(hipGetDeviceProperties(&prop, dev));
+
+        std::cout << "============================================================\n";
+        std::cout << "Device " << dev << ": " << prop.name << "\n";
+        std::cout << "============================================================\n";
+
+        // --- Device Properties ---
+        std::cout << "\n[Device Properties]\n";
+        std::cout << std::left << std::setw(28) << "major.minor"
+                  << ": " << prop.major << "." << prop.minor << "\n";
+
+// AMD-specific helpful fields if present in your HIP version:
+// prop.gcnArchName may exist; print defensively using std::string
+#if defined(__HIP_PLATFORM_AMD__)
+        if(!std::string_view{prop.gcnArchName}.empty())
+        {
+            std::cout << std::left << std::setw(28) << "gcnArchName"
+                      << ": " << prop.gcnArchName << "\n";
+        }
+#endif
+
+        printBytes("totalGlobalMem", prop.totalGlobalMem);
+        printBytes("sharedMemPerBlock", prop.sharedMemPerBlock);
+        printBytes("sharedMemPerMultiprocessor", prop.sharedMemPerMultiprocessor);
+        std::cout << std::left << std::setw(28) << "regsPerBlock"
+                  << ": " << prop.regsPerBlock << "\n";
+        std::cout << std::left << std::setw(28) << "warpSize/wavefront"
+                  << ": " << prop.warpSize << "\n";
+        std::cout << std::left << std::setw(28) << "maxThreadsPerBlock"
+                  << ": " << prop.maxThreadsPerBlock << "\n";
+        printArray3("maxThreadsDim", prop.maxThreadsDim);
+        printArray3("maxGridSize", prop.maxGridSize);
+        std::cout << std::left << std::setw(28) << "clockRate (kHz)"
+                  << ": " << prop.clockRate << "\n";
+        std::cout << std::left << std::setw(28) << "memoryClockRate (kHz)"
+                  << ": " << prop.memoryClockRate << "\n";
+        std::cout << std::left << std::setw(28) << "memoryBusWidth (bits)"
+                  << ": " << prop.memoryBusWidth << "\n";
+        printBytes("totalConstMem", prop.totalConstMem);
+        std::cout << std::left << std::setw(28) << "multiProcessorCount"
+                  << ": " << prop.multiProcessorCount << "\n";
+        std::cout << std::left << std::setw(28) << "l2CacheSize (bytes)"
+                  << ": " << prop.l2CacheSize << "\n";
+        std::cout << std::left << std::setw(28) << "maxThreadsPerMultiProcessor"
+                  << ": " << prop.maxThreadsPerMultiProcessor << "\n";
+        printBool("concurrentKernels", prop.concurrentKernels);
+        printBool("cooperativeLaunch", prop.cooperativeLaunch);
+        printBool("cooperativeMultiDeviceLaunch", prop.cooperativeMultiDeviceLaunch);
+        printBool("managedMemory", prop.managedMemory);
+        printBool("pageableMemoryAccess", prop.pageableMemoryAccess);
+        printBool("concurrentManagedAccess", prop.concurrentManagedAccess);
+        printIfNonzero("pciDomainID", prop.pciDomainID);
+        printIfNonzero("pciBusID", prop.pciBusID);
+        printIfNonzero("pciDeviceID", prop.pciDeviceID);
+        std::cout << std::left << std::setw(28) << "isMultiGpuBoard"
+                  << ": " << (prop.isMultiGpuBoard != 0 ? "true" : "false") << "\n";
+
+        // --- Device Flags / Cache / SharedMem config are per current device context ---
+        std::cout << "\n[Device Flags]\n";
+        unsigned int flags = 0;
+        HIP_CHECK(hipGetDeviceFlags(&flags));
+        std::cout << "flags (hex): 0x" << std::hex << std::setw(8) << std::setfill('0') << flags
+                  << std::dec << std::setfill(' ') << "\n";
+
+// Optionally decode some common bits if present in your HIP:
+#ifdef hipDeviceScheduleAuto
+        if((flags & hipDeviceScheduleAuto) != 0u) std::cout << " - hipDeviceScheduleAuto\n";
+#endif
+#ifdef hipDeviceScheduleSpin
+        if((flags & hipDeviceScheduleSpin) != 0u) std::cout << " - hipDeviceScheduleSpin\n";
+#endif
+#ifdef hipDeviceScheduleBlockingSync
+        if((flags & hipDeviceScheduleBlockingSync) != 0u)
+            std::cout << " - hipDeviceScheduleBlockingSync\n";
+#endif
+#ifdef hipDeviceMapHost
+        if((flags & hipDeviceMapHost) != 0u) std::cout << " - hipDeviceMapHost\n";
+#endif
+#ifdef hipDeviceLmemResizeToMax
+        if((flags & hipDeviceLmemResizeToMax) != 0u) std::cout << " - hipDeviceLmemResizeToMax\n";
+#endif
+
+        std::cout << "\n[Device Cache Config]\n";
+        hipFuncCache_t cacheCfg{};
+        HIP_CHECK(hipDeviceGetCacheConfig(&cacheCfg));
+        std::cout << "cache config: " << funcCacheToStr(cacheCfg) << "\n";
+
+        std::cout << "\n[Shared Memory Config]\n";
+        hipSharedMemConfig shCfg{};
+        HIP_CHECK(hipDeviceGetSharedMemConfig(&shCfg));
+        std::cout << "shared mem bank size: " << shmemBankToStr(shCfg) << "\n";
+
+        // --- Device Limits ---
+        std::cout << "\n[Device Limits]\n";
+        for(const auto& ld : kLimits)
+        {
+            size_t     value = 0;
+            hipError_t le    = hipDeviceGetLimit(&value, ld.limit);
+            if(le == hipSuccess)
+            {
+                if(ld.limit == hipLimitPrintfFifoSize || ld.limit == hipLimitMallocHeapSize ||
+                   ld.limit == hipLimitStackSize)
+                {
+                    printBytes(ld.name, value);
+                }
+                else
+                {
+                    std::cout << std::left << std::setw(28) << ld.name << ": " << value << "\n";
+                }
+            }
+            else if(le == hipErrorUnsupportedLimit)
+            {
+                std::cout << std::left << std::setw(28) << ld.name << ": unsupported\n";
+            }
+            else
+            {
+                std::cout << std::left << std::setw(28) << ld.name << ": error ("
+                          << hipGetErrorName(le) << ")\n";
+            }
+        }
+
+        std::cout << std::endl;
+    }
+
+    return 0;
+}

projects/rocprofiler-sdk/tests/hip-host-tracing/CMakeLists.txt

@@ -0,0 +1,60 @@
+#
+#
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+project(
+    rocprofiler-sdk-tests-hip-host-tracing
+    LANGUAGES CXX
+    VERSION 0.0.0)
+
+find_package(rocprofiler-sdk REQUIRED)
+
+if(ROCPROFILER_MEMCHECK_PRELOAD_ENV)
+    set(PRELOAD_ENV
+        "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}:$<TARGET_FILE:rocprofiler-sdk-json-tool>")
+else()
+    set(PRELOAD_ENV "LD_PRELOAD=$<TARGET_FILE:rocprofiler-sdk-json-tool>")
+endif()
+
+add_test(NAME test-hip-host-tracing-execute COMMAND $<TARGET_FILE:hip-host>)
+
+set(hip-host-tracing-env
+    "${PRELOAD_ENV}"
+    "ROCPROFILER_TOOL_OUTPUT_FILE=hip-host-tracing-test.json"
+    "LD_LIBRARY_PATH=$<TARGET_FILE_DIR:rocprofiler-sdk::rocprofiler-sdk-shared-library>:$ENV{LD_LIBRARY_PATH}"
+    )
+
+set_tests_properties(
+    test-hip-host-tracing-execute
+    PROPERTIES TIMEOUT
+               100
+               LABELS
+               "integration-tests"
+               ENVIRONMENT
+               "${hip-host-tracing-env}"
+               FAIL_REGULAR_EXPRESSION
+               "${ROCPROFILER_DEFAULT_FAIL_REGEX}"
+               FIXTURES_SETUP
+               hip-host-tracing)
+
+# copy to binary directory
+rocprofiler_configure_pytest_files(COPY validate.py conftest.py CONFIG pytest.ini)
+
+add_test(
+    NAME test-hip-host-tracing-validate
+    COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py ${PYTEST_ARGS}
+            --input ${CMAKE_CURRENT_BINARY_DIR}/hip-host-tracing-test.json)
+
+set_tests_properties(
+    test-hip-host-tracing-validate
+    PROPERTIES TIMEOUT
+               45
+               LABELS
+               "integration-tests"
+               DEPENDS
+               test-hip-host-tracing-execute
+               FAIL_REGULAR_EXPRESSION
+               "${ROCPROFILER_DEFAULT_FAIL_REGEX}"
+               FIXTURES_REQUIRED
+               hip-host-tracing)

projects/rocprofiler-sdk/tests/hip-host-tracing/conftest.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+
+# MIT License
+#
+# Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import json
+import pytest
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--input",
+        action="store",
+        default="hip-host-tracing-test.json",
+        help="Input JSON",
+    )
+
+
+@pytest.fixture
+def input_data(request):
+    filename = request.config.getoption("--input")
+    with open(filename, "r") as inp:
+        return json.load(inp)

projects/rocprofiler-sdk/tests/hip-host-tracing/pytest.ini

@@ -0,0 +1,5 @@
+
+[pytest]
+addopts = --durations=20 -rA -s -vv
+testpaths = validate.py
+pythonpath = @ROCPROFILER_SDK_TESTS_BINARY_DIR@/pytest-packages

projects/rocprofiler-sdk/tests/hip-host-tracing/validate.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+
+# MIT License
+#
+# Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import sys
+import pytest
+
+test_api_traces = [
+    "hip_api_traces",
+    "hsa_api_traces",
+    "marker_api_traces",
+    "rccl_api_traces",
+]
+
+
+# helper function
+def node_exists(name, data, min_len=1):
+    assert name in data
+    assert data[name] is not None
+    if isinstance(data[name], (list, tuple, dict, set)):
+        assert len(data[name]) >= min_len
+
+
+def test_data_structure(input_data):
+    """verify minimum amount of expected data is present"""
+    data = input_data
+
+    node_exists("rocprofiler-sdk-json-tool", data)
+
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    node_exists("metadata", sdk_data)
+    node_exists("pid", sdk_data["metadata"])
+    node_exists("main_tid", sdk_data["metadata"])
+    node_exists("init_time", sdk_data["metadata"])
+    node_exists("fini_time", sdk_data["metadata"])
+
+    node_exists("agents", sdk_data)
+    node_exists("call_stack", sdk_data)
+    node_exists("callback_records", sdk_data)
+    node_exists("buffer_records", sdk_data)
+
+    node_exists("names", sdk_data["callback_records"])
+    node_exists("names", sdk_data["buffer_records"])
+
+    for titr in test_api_traces:
+        min_len = (
+            1
+            if titr
+            in [
+                "hip_api_traces",
+                "hsa_api_traces",
+            ]
+            else 0
+        )
+        node_exists(titr, sdk_data["callback_records"], min_len)
+        node_exists(titr, sdk_data["buffer_records"], min_len)
+
+    node_exists("retired_correlation_ids", sdk_data["buffer_records"])
+
+
+def test_timestamps(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    cb_start = {}
+    cb_end = {}
+    for titr in test_api_traces:
+        for itr in sdk_data["callback_records"][titr]:
+            cid = itr["correlation_id"]["internal"]
+            phase = itr["phase"]
+            if phase == 1:
+                cb_start[cid] = itr["timestamp"]
+            elif phase == 2:
+                cb_end[cid] = itr["timestamp"]
+                assert cb_start[cid] <= itr["timestamp"]
+            else:
+                assert phase == 1 or phase == 2
+
+        for itr in sdk_data["buffer_records"][titr]:
+            assert itr["start_timestamp"] <= itr["end_timestamp"]
+
+
+def test_retired_correlation_ids(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    def _sort_dict(inp):
+        return dict(sorted(inp.items()))
+
+    api_corr_ids = {}
+    for titr in test_api_traces:
+        for itr in sdk_data["buffer_records"][titr]:
+            corr_id = itr["correlation_id"]["internal"]
+            assert corr_id not in api_corr_ids.keys()
+            api_corr_ids[corr_id] = itr
+
+    retired_corr_ids = {}
+    for itr in sdk_data["buffer_records"]["retired_correlation_ids"]:
+        corr_id = itr["internal_correlation_id"]
+        assert corr_id not in retired_corr_ids.keys()
+        retired_corr_ids[corr_id] = itr
+
+    api_corr_ids = _sort_dict(api_corr_ids)
+    retired_corr_ids = _sort_dict(retired_corr_ids)
+
+    for cid, itr in api_corr_ids.items():
+        assert cid in retired_corr_ids.keys()
+        retired_ts = retired_corr_ids[cid]["timestamp"]
+        end_ts = itr["end_timestamp"]
+        assert (retired_ts - end_ts) > 0, f"correlation-id: {cid}, data: {itr}"
+
+    assert len(api_corr_ids.keys()) == (len(retired_corr_ids.keys()))
+
+
+if __name__ == "__main__":
+    exit_code = pytest.main(["-x", __file__] + sys.argv[1:])
+    sys.exit(exit_code)