Shared Library Constructor (rocprofv3 deadlock fix) (#599)

* Moved tests/apps to tests/bin

* Renamed cmake project in tests/bin

* Update samples

- Use ROCPROFILER_DEFAULT_FAIL_REGEX
- tweaks to stdout messages

* Update tests

- Use ROCPROFILER_DEFAULT_FAIL_REGEX

* Add tests/lib

- libraries with HIP code

* Update PTL submodule

- remove atexit delete of thread_id_map

* Update cmake/rocprofiler_options.cmake

- Set ROCPROFILER_DEFAULT_FAIL_REGEX

* Update common lib: env + logging

- improved customization of logging settings
- default to disabling logging to files
- install failure handler for rocprofv3
- set_env support in environment.*

* Add lib/rocprofiler-sdk/shared_library.cpp

- shared library constructor

* Update lib/rocprofiler-sdk-tool/tool.cpp

- destructor thread safety
- convert callback_name_info and buffered_name_info to pointers
- install failure handler for logging

* Add tests/bin/hip-in-libraries

- hip-in-libraries is an exe which uses two shared libraries where each shared library contains HIP kernels
  - used for testing deadlocking within __hipRegisterFatBinary

* Update bin/rocprofv3

- reorganized the env variables
- use exec to launch command
- set ROCPROFILER_LIBRARY_CTOR=1

* Add tests/rocprofv3/tracing-hip-in-libraries

- uses hip-in-libraries exe for exe which uses shared libraries to launch HIP kernels

* Update bin/rocprofv3

- fix counter collection (no exec)

* Update lib/rocprofiler-sdk-tool/tool.cpp

- replace "Kernel-Name" with "Kernel_Name"

* Update lib/rocprofiler-sdk/registration.cpp

Use RTLD_LOCAL instead of RTLD_GLOBAL for env libraries

* Update tests/rocprofv3

- replace "Kernel-Name" with "Kernel_Name"

* Update tests

- vector-ops (bin) stream syncs + runs with 4 queues per device
- improve counter-collection/input1 validation
- rocprofv3/tracing-hip-in-libraries does not do sys-trace
- improved validation script for tracing-hip-in-libraries
- updated dispatch_callback in json-tool.cpp following reworking of prototypes for counter collection

* Update samples/counter_collection

- updated dispatch_callback(s) and record_callback(s) following reworking of prototypes

* Update bin/rocprofv3

- reorganized help menu
- added options for sub-HSA tables
- added --hip-runtime-trace
- changed --hip-trace to include --hip-compiler-trace

* Update lib/rocprofiler-sdk-tool

- improved kernel filtering
- removed arch_vgpr, accum_vgpr, sgpr code (in rocprofiler-sdk)
- fixed issue with counter-collection w/o tracing
- added support for fine grained HSA API tracing
- removed directly linking to HSA-runtime

* Update lib/rocprofiler-sdk/agent.cpp

- rocp_agents != hsa_agents is non-fatal when ROCPROFILER_BUILD_CI=OFF (CMake option)

* GPR (vector and scalar) info in kernel symbol data

- rocprofiler_callback_tracing_code_object_kernel_symbol_register_data_t contains general purpose register info

* Header include order fix

- Include repo headers first
- Third party library headers next
- standard library headers last

* Update dispatch profiling public API

- introduce rocprofiler_profile_counting_dispatch_data_t
- change signature of rocprofiler_profile_counting_dispatch_callback_t and rocprofiler_profile_counting_record_callback_t
- provide rocprofiler_user_data_t pointer in dispatch callback
- provide rocprofiler_user_data_t value (from dispatch cb) in record callback

* Update tests/bin/CMakeLists.txt

- fix add_subdirectory(hip-in-libraries) order

* Update VERSION

- bump to 0.2.0 in prep for AFAR
This commit is contained in:
Jonathan R. Madsen
2024-03-07 22:21:26 -06:00
committed by GitHub
parent 665c546e65
commit 7b6d3c70bd
85 changed files with 2497 additions and 856 deletions
+2 -1
View File
@@ -24,5 +24,6 @@ enable_testing()
include(CTest)
add_subdirectory(tracing)
add_subdirectory(counter-collection)
add_subdirectory(tracing-plus-cc)
add_subdirectory(tracing-hip-in-libraries)
add_subdirectory(counter-collection)
@@ -1,3 +1,7 @@
#
# Various counter collection tests
#
foreach(FILENAME conftest.py pytest.ini)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/${FILENAME}
${CMAKE_CURRENT_BINARY_DIR}/${FILENAME} COPYONLY)
@@ -40,7 +40,7 @@ set(cc-env-pmc1
set_tests_properties(
rocprofv3-test-counter-collection-pmc1-execute
PROPERTIES TIMEOUT 45 LABELS "integration-tests" ENVIRONMENT "${cc-env-pmc1}"
FAIL_REGULAR_EXPRESSION "threw an exception")
FAIL_REGULAR_EXPRESSION "${ROCPROFILER_DEFAULT_FAIL_REGEX}")
add_test(NAME rocprofv3-test-counter-collection-pmc1-validate
COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --input
@@ -50,4 +50,4 @@ set_tests_properties(
rocprofv3-test-counter-collection-pmc1-validate
PROPERTIES TIMEOUT 45 LABELS "integration-tests" DEPENDS
rocprofv3-test-counter-collection-pmc1-execute FAIL_REGULAR_EXPRESSION
"threw an exception")
"${ROCPROFILER_DEFAULT_FAIL_REGEX}")
@@ -1,30 +1,46 @@
import pandas as pd
#!/usr/bin/env python3
import sys
import pytest
import numpy as np
import pandas as pd
kernel_list = ["addition_kernel", "subtract_kernel", "multiply_kernel", "divide_kernel"]
kernel_list = sorted(
["addition_kernel", "subtract_kernel", "multiply_kernel", "divide_kernel"]
)
def unique(lst):
return list(set(lst))
def test_validate_counter_collection_pmc1(input_data: pd.DataFrame):
df = input_data
assert df.empty == False
assert not df.empty
assert (df["Agent_Id"].astype(int).values > 0).all()
assert (df["Queue_Id"].astype(int).values > 0).all()
assert (df["Process_Id"].astype(int).values > 0).all()
assert len(df["Kernel-Name"]) > 0
df_list = df["Kernel-Name"].values.flatten().tolist()
# Check if each string in kernel_list is present at least once
missing_kernels = []
for kernel in kernel_list:
if kernel not in df_list:
missing_kernels.append(kernel)
assert len(df["Kernel_Name"]) > 0
assert kernel_list == sorted(df["Kernel_Name"].unique().tolist())
kernel_count = dict([[itr, 0] for itr in kernel_list])
assert len(kernel_count) == len(kernel_list)
for itr in df["Kernel_Name"]:
kernel_count[itr] += 1
kn_cnt = [itr for _, itr in kernel_count.items()]
assert min(kn_cnt) == max(kn_cnt) and len(unique(kn_cnt)) == 1
assert (
not missing_kernels
), f"The following kernel names are missing from the out file: {missing_kernels}"
assert df["Counter_Name"].str.contains("SQ_WAVES").all()
assert len(df["Counter_Value"]) > 0
assert df["Counter_Name"].str.contains("SQ_WAVES").all()
assert (df["Counter_Value"].astype(int).values > 0).all()
di_list = df["Dispatch_Id"].astype(int).values.tolist()
di_uniq = sorted(df["Dispatch_Id"].unique().tolist())
# make sure the dispatch ids are unique and ordered
di_expect = [idx + 1 for idx in range(len(di_list))]
assert di_expect == di_uniq
if __name__ == "__main__":
@@ -40,7 +40,7 @@ set(cc-env-pmc2
set_tests_properties(
rocprofv3-test-counter-collection-pmc2-execute
PROPERTIES TIMEOUT 45 LABELS "integration-tests" ENVIRONMENT "${cc-env-pmc2}"
FAIL_REGULAR_EXPRESSION "threw an exception")
FAIL_REGULAR_EXPRESSION "${ROCPROFILER_DEFAULT_FAIL_REGEX}")
add_test(NAME rocprofv3-test-counter-collection-pmc2-validate
COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py
@@ -50,4 +50,4 @@ set_tests_properties(
rocprofv3-test-counter-collection-pmc2-validate
PROPERTIES TIMEOUT 45 LABELS "integration-tests" DEPENDS
rocprofv3-test-counter-collection-pmc2-execute FAIL_REGULAR_EXPRESSION
"threw an exception")
"${ROCPROFILER_DEFAULT_FAIL_REGEX}")
@@ -34,7 +34,7 @@ def test_validate_counter_collection_pmc2(input_dir: pd.DataFrame):
with open(file_path, "r") as file:
df = pd.read_csv(file)
# check if kernel-name is present
assert len(df["Kernel-Name"]) > 0
assert len(df["Kernel_Name"]) > 0
# check if counter value is positive
assert len(df["Counter_Value"]) > 0
@@ -41,7 +41,7 @@ set(cc-env-list-metrics
set_tests_properties(
rocprofv3-test-list-metrics-execute
PROPERTIES TIMEOUT 45 LABELS "integration-tests" ENVIRONMENT "${cc-env-list-metrics}"
FAIL_REGULAR_EXPRESSION "threw an exception")
FAIL_REGULAR_EXPRESSION "${ROCPROFILER_DEFAULT_FAIL_REGEX}")
set_tests_properties(
rocprofv3-test-list-metrics-std-out-execute
@@ -76,6 +76,6 @@ set_tests_properties(
DEPENDS
rocprofv3-test-list-metrics-execute
FAIL_REGULAR_EXPRESSION
"threw an exception"
"${ROCPROFILER_DEFAULT_FAIL_REGEX}"
ATTACHED_FILES_ON_FAIL
"${VALIDATION_FILES}")
@@ -3,3 +3,4 @@
addopts = --durations=20 -rA -s -vv
testpaths = input1/validate.py
input2/validate.py
list_metrics/validate.py
@@ -0,0 +1,72 @@
#
# rocprofv3 tool test
#
cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
project(
rocprofiler-tests-rocprofv3-tracing-hip-in-libraries
LANGUAGES CXX
VERSION 0.0.0)
find_package(rocprofiler-sdk REQUIRED)
add_test(
NAME rocprofv3-test-trace-hip-in-libraries-execute
COMMAND
$<TARGET_FILE:rocprofiler-sdk::rocprofv3> --hip-runtime-trace
--hip-compiler-trace --hsa-core-trace --hsa-amd-trace --hsa-image-trace
--hsa-finalizer-trace --kernel-trace --memory-copy-trace -d
${CMAKE_CURRENT_BINARY_DIR}/%argt%-trace -o out $<TARGET_FILE:hip-in-libraries>)
string(REPLACE "LD_PRELOAD=" "ROCPROF_PRELOAD=" PRELOAD_ENV
"${ROCPROFILER_MEMCHECK_PRELOAD_ENV}")
set(tracing-env "${PRELOAD_ENV}"
"HSA_TOOLS_LIB=$<TARGET_FILE:rocprofiler::rocprofiler-shared-library>")
set_tests_properties(
rocprofv3-test-trace-hip-in-libraries-execute
PROPERTIES
LABELS
"integration-tests"
ENVIRONMENT
"${tracing-env}"
FAIL_REGULAR_EXPRESSION
"HSA_CORE_API|HSA_AMD_EXT_API|HSA_IMAGE_EXT_API|HSA_FINALIZER_EXT_API|HIP_API|HIP_COMPILER_API|KERNEL_DISPATCH|CODE_OBJECT"
)
foreach(FILENAME validate.py conftest.py)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/${FILENAME}
${CMAKE_CURRENT_BINARY_DIR}/${FILENAME} COPYONLY)
endforeach()
add_test(
NAME rocprofv3-test-trace-hip-in-libraries-validate
COMMAND
${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --hsa-input
${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_hsa_api_trace.csv
--hip-input
${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_hip_api_trace.csv
--kernel-input
${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_kernel_trace.csv
--memory-copy-input
${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_memory_copy_trace.csv)
set(VALIDATION_FILES
${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_memory_copy_trace.csv
${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_hsa_api_trace.csv
${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_hip_api_trace.csv
${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_kernel_trace.csv)
set_tests_properties(
rocprofv3-test-trace-hip-in-libraries-validate
PROPERTIES TIMEOUT
45
LABELS
"integration-tests"
DEPENDS
rocprofv3-test-trace-hip-in-libraries-execute
FAIL_REGULAR_EXPRESSION
"AssertionError"
ATTACHED_FILES_ON_FAIL
"${VALIDATION_FILES}")
@@ -0,0 +1,94 @@
#!/usr/bin/env python3
import os
import csv
import pytest
def pytest_addoption(parser):
parser.addoption(
"--hsa-input",
action="store",
help="Path to HSA API tracing CSV file.",
)
parser.addoption(
"--kernel-input",
action="store",
help="Path to kernel tracing CSV file.",
)
parser.addoption(
"--memory-copy-input",
action="store",
help="Path to memory-copy tracing CSV file.",
)
parser.addoption(
"--marker-input",
action="store",
help="Path to marker API tracing CSV file.",
)
parser.addoption(
"--hip-input",
action="store",
help="Path to HIP runtime and compiler API tracing CSV file.",
)
@pytest.fixture
def hsa_input_data(request):
filename = request.config.getoption("--hsa-input")
data = []
with open(filename, "r") as inp:
reader = csv.DictReader(inp)
for row in reader:
data.append(row)
return data
@pytest.fixture
def kernel_input_data(request):
filename = request.config.getoption("--kernel-input")
data = []
with open(filename, "r") as inp:
reader = csv.DictReader(inp)
for row in reader:
data.append(row)
return data
@pytest.fixture
def memory_copy_input_data(request):
filename = request.config.getoption("--memory-copy-input")
data = []
with open(filename, "r") as inp:
reader = csv.DictReader(inp)
for row in reader:
data.append(row)
return data
@pytest.fixture
def marker_input_data(request):
filename = request.config.getoption("--marker-input")
data = []
with open(filename, "r") as inp:
reader = csv.DictReader(inp)
for row in reader:
data.append(row)
return data
@pytest.fixture
def hip_input_data(request):
filename = request.config.getoption("--hip-input")
data = []
if os.path.exists(filename):
with open(filename, "r") as inp:
reader = csv.DictReader(inp)
for row in reader:
data.append(row)
return data
@@ -0,0 +1,142 @@
#!/usr/bin/env python3
import sys
import pytest
class dim3(object):
def __init__(self, x, y, z):
self.x = int(x)
self.y = int(y)
self.z = int(z)
def as_tuple(self):
return (self.x, self.y, self.z)
def test_api_trace(hsa_input_data, hip_input_data):
functions = []
correlation_ids = []
for row in hsa_input_data:
assert row["Domain"] in (
"HSA_CORE_API",
"HSA_AMD_EXT_API",
"HSA_IMAGE_EXT_API",
"HSA_FINALIZE_EXT_API",
)
assert int(row["Process_Id"]) > 0
assert int(row["Thread_Id"]) >= int(row["Process_Id"])
assert int(row["End_Timestamp"]) >= int(row["Start_Timestamp"])
functions.append(row["Function"])
correlation_ids.append(int(row["Correlation_Id"]))
for row in hip_input_data:
assert row["Domain"] in [
"HIP_RUNTIME_API",
"HIP_COMPILER_API",
]
assert int(row["Process_Id"]) > 0
assert int(row["Thread_Id"]) == 0 or int(row["Thread_Id"]) >= int(
row["Process_Id"]
)
assert int(row["End_Timestamp"]) >= int(row["Start_Timestamp"])
functions.append(row["Function"])
correlation_ids.append(int(row["Correlation_Id"]))
correlation_ids = sorted(list(set(correlation_ids)))
# all correlation ids are unique
assert len(correlation_ids) == (len(hsa_input_data) + len(hip_input_data))
# correlation ids are numbered from 1 to N
assert correlation_ids[0] == 1
assert correlation_ids[-1] == len(correlation_ids)
functions = list(set(functions))
for itr in (
"hsa_amd_memory_async_copy_on_engine",
"hsa_agent_get_info",
"hsa_agent_iterate_isas",
"hsa_signal_create",
"hsa_agent_get_info",
"hsa_executable_symbol_get_info",
):
assert itr in functions
if hip_input_data:
for itr in (
"hipGetLastError",
"hipLaunchKernel",
"hipStreamSynchronize",
"hipMemcpyAsync",
"hipFree",
"hipStreamDestroy",
"hipDeviceSynchronize",
"hipDeviceReset",
"hipSetDevice",
):
assert itr in functions
def test_kernel_trace(kernel_input_data):
valid_kernel_names = sorted(
[
"__amd_rocclr_fillBufferAligned",
"(anonymous namespace)::transpose(int const*, int*, int, int)",
"void (anonymous namespace)::addition_kernel<float>(float*, float const*, float const*, int, int)",
"void (anonymous namespace)::divide_kernel<float>(float*, float const*, float const*, int, int)",
"void (anonymous namespace)::multiply_kernel<float>(float*, float const*, float const*, int, int)",
"void (anonymous namespace)::subtract_kernel<float>(float*, float const*, float const*, int, int)",
]
)
kernels = []
for row in kernel_input_data:
kernel_name = row["Kernel_Name"]
assert row["Kind"] == "KERNEL_DISPATCH"
assert int(row["Agent_Id"]) > 0
assert int(row["Queue_Id"]) > 0
assert int(row["Kernel_Id"]) > 0
assert int(row["Correlation_Id"]) > 0
assert kernel_name in valid_kernel_names
if kernel_name not in kernels:
kernels.append(kernel_name)
workgrp_size = dim3(
row["Workgroup_Size_X"], row["Workgroup_Size_Y"], row["Workgroup_Size_Z"]
)
grid_size = dim3(row["Grid_Size_X"], row["Grid_Size_Y"], row["Grid_Size_Z"])
if kernel_name == "__amd_rocclr_fillBufferAligned":
assert workgrp_size.as_tuple() > (1, 1, 1)
assert grid_size.as_tuple() > (1, 1, 1)
elif "transpose" in kernel_name:
assert workgrp_size.as_tuple() == (32, 32, 1)
assert grid_size.as_tuple() == (9920, 9920, 1)
else:
assert workgrp_size.as_tuple() == (64, 1, 1)
assert grid_size.as_tuple() == (4096, 2048, 1)
assert int(row["End_Timestamp"]) >= int(row["Start_Timestamp"])
kernels = sorted(list(set(kernels)))
assert kernels == valid_kernel_names
def test_memory_copy_trace(memory_copy_input_data):
for row in memory_copy_input_data:
assert row["Kind"] == "MEMORY_COPY"
assert row["Direction"] in ("HOST_TO_DEVICE", "DEVICE_TO_HOST")
if row["Direction"] == "HOST_TO_DEVICE":
assert int(row["Source_Agent_Id"]) == 0
elif row["Direction"] == "DEVICE_TO_HOST":
assert int(row["Destination_Agent_Id"]) == 0
assert int(row["Correlation_Id"]) > 0
assert int(row["End_Timestamp"]) >= int(row["Start_Timestamp"])
assert len(memory_copy_input_data) == 120
if __name__ == "__main__":
exit_code = pytest.main(["-x", __file__] + sys.argv[1:])
sys.exit(exit_code)
@@ -40,7 +40,7 @@ set(cc-tracing-env
set_tests_properties(
rocprofv3-test-tracing-plus-cc-execute
PROPERTIES TIMEOUT 45 LABELS "integration-tests" ENVIRONMENT "${cc-tracing-env}"
FAIL_REGULAR_EXPRESSION "threw an exception")
FAIL_REGULAR_EXPRESSION "${ROCPROFILER_DEFAULT_FAIL_REGEX}")
add_test(NAME rocprofv3-test-tracing-plus-cc-validate
COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py
@@ -50,4 +50,4 @@ set_tests_properties(
rocprofv3-test-tracing-plus-cc-validate
PROPERTIES TIMEOUT 45 LABELS "integration-tests" DEPENDS
rocprofv3-test-tracing-plus-cc-execute FAIL_REGULAR_EXPRESSION
"threw an exception")
"${ROCPROFILER_DEFAULT_FAIL_REGEX}")
+1 -1
View File
@@ -34,7 +34,7 @@ def test_validate_counter_collection_plus_tracing(input_dir: pd.DataFrame):
with open(file_path, "r") as file:
df = pd.read_csv(file)
# check if either kernel-name/FUNCTION is present
assert "Kernel-Name" in df.columns or "Function" in df.columns
assert "Kernel_Name" in df.columns or "Function" in df.columns
if __name__ == "__main__":
+1 -1
View File
@@ -23,7 +23,7 @@ def test_hsa_api_trace(hsa_input_data):
correlation_ids = sorted(list(set(correlation_ids)))
hsa_api_calls_offset = 2 # roctxRangePush is first
num_marker_api_calls = 6 # seven marker API calls, only six entries in
num_marker_api_calls = 7 # seven marker API calls, only six entries in
# marker csv data because roctxRangePush + roctxRangePop is one entry
# all correlation ids are unique