From 5e4e7b41f161da46af77ace1b608416280361992 Mon Sep 17 00:00:00 2001 From: "Jonathan R. Madsen" Date: Wed, 20 Sep 2023 23:14:25 -0500 Subject: [PATCH] Documentation, sanitizers, and PTL submodule (#71) * Update scripts/thread-sanitizer-suppr.txt - ignore data race occasionally triggered by libamdhip64.so * Update external/CMakeLists.txt - configure PTL to use locks in task queues * Update PTL submodule - tweal to task queues to prevent data race from std::list next pointer * Add scripts/setup-sanitizer-env.sh - bash script that exports the {ASAN,LSAN,TSAN}_OPTIONS used by run-ci.py * Update include/rocprofiler (doxygen) - fix doxygen grouping * Update docs workflow - change concurrency group to be specific to workflow + ref - this prevents separate PRs triggering this workflow from cancelling each other --- .github/workflows/docs.yml | 2 +- external/CMakeLists.txt | 3 +- external/ptl | 2 +- source/include/rocprofiler/agent.h | 5 +- source/include/rocprofiler/agent_profile.h | 5 +- source/include/rocprofiler/buffer.h | 5 +- source/include/rocprofiler/buffer_tracing.h | 6 +- source/include/rocprofiler/callback_tracing.h | 6 +- source/include/rocprofiler/context.h | 5 +- source/include/rocprofiler/counters.h | 4 +- source/include/rocprofiler/defines.h | 5 +- source/include/rocprofiler/dispatch_profile.h | 6 +- .../rocprofiler/external_correlation.h | 3 +- source/include/rocprofiler/fwd.h | 4 +- .../include/rocprofiler/internal_threading.h | 6 +- source/include/rocprofiler/pc_sampling.h | 4 +- source/include/rocprofiler/profile_config.h | 4 +- source/include/rocprofiler/rocprofiler.h | 57 ++++++++----- .../include/rocprofiler/rocprofiler_plugin.h | 85 ++++++++++--------- source/include/rocprofiler/spm.h | 5 +- source/scripts/setup-sanitizer-env.sh | 38 +++++++++ source/scripts/thread-sanitizer-suppr.txt | 3 + 22 files changed, 171 insertions(+), 92 deletions(-) create mode 100755 source/scripts/setup-sanitizer-env.sh diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 7945066546..3a33ff52da 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -22,7 +22,7 @@ on: - '.github/workflows/docs.yml' concurrency: - group: "pages" + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true env: diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index b67a0e2dc3..761889f23c 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -96,9 +96,10 @@ if(NOT TARGET PTL::ptl-static) REPO_URL https://github.com/jrmadsen/PTL.git REPO_BRANCH rocprofiler) - set(PTL_BUILD_EXAMPLES OFF) set(PTL_USE_TBB OFF) set(PTL_USE_GPU OFF) + set(PTL_USE_LOCKS ON) + set(PTL_BUILD_EXAMPLES OFF) set(PTL_DEVELOPER_INSTALL OFF) if(NOT DEFINED BUILD_OBJECT_LIBS) diff --git a/external/ptl b/external/ptl index 7bbc5a4e66..43c3937db3 160000 --- a/external/ptl +++ b/external/ptl @@ -1 +1 @@ -Subproject commit 7bbc5a4e66d10d7acae5b353838e2404b3dd3742 +Subproject commit 43c3937db3b57b94bd1995ec7249a083ef0cde48 diff --git a/source/include/rocprofiler/agent.h b/source/include/rocprofiler/agent.h index f45dfc9030..c54c665e9a 100644 --- a/source/include/rocprofiler/agent.h +++ b/source/include/rocprofiler/agent.h @@ -27,7 +27,10 @@ ROCPROFILER_EXTERN_C_INIT -/** @defgroup AGENTS Agent Information +/** + * @defgroup AGENTS Agent Information + * @brief needs brief description + * * @{ */ diff --git a/source/include/rocprofiler/agent_profile.h b/source/include/rocprofiler/agent_profile.h index 819f178f66..533f979a43 100644 --- a/source/include/rocprofiler/agent_profile.h +++ b/source/include/rocprofiler/agent_profile.h @@ -27,7 +27,10 @@ ROCPROFILER_EXTERN_C_INIT -/** @defgroup AGENT_PROFILE_COUNTING_SERVICE Agent Profile Counting Service +/** + * @defgroup AGENT_PROFILE_COUNTING_SERVICE Agent Profile Counting Service + * @brief needs brief description + * * @{ */ diff --git a/source/include/rocprofiler/buffer.h b/source/include/rocprofiler/buffer.h index cd7f1a1af1..b99da626f9 100644 --- a/source/include/rocprofiler/buffer.h +++ b/source/include/rocprofiler/buffer.h @@ -27,7 +27,10 @@ ROCPROFILER_EXTERN_C_INIT -/** @defgroup BUFFER_HANDLING Buffer +/** + * @defgroup BUFFER_HANDLING Buffer Handling + * @brief Creation, destruction, and flushing of buffers populated with data from rocprofiler + * * @{ * * Every Buffer is associated with a specific service kind. diff --git a/source/include/rocprofiler/buffer_tracing.h b/source/include/rocprofiler/buffer_tracing.h index 275659e3ed..05df9c4af1 100644 --- a/source/include/rocprofiler/buffer_tracing.h +++ b/source/include/rocprofiler/buffer_tracing.h @@ -27,9 +27,9 @@ ROCPROFILER_EXTERN_C_INIT -/** @defgroup BUFFER_TRACING_SERVICE Asynchronous Tracing Service - * - * Receive callbacks for batches of records from an internal (background) thread +/** + * @defgroup BUFFER_TRACING_SERVICE Asynchronous Tracing Service + * @brief Receive callbacks for batches of records from an internal (background) thread * * @{ */ diff --git a/source/include/rocprofiler/callback_tracing.h b/source/include/rocprofiler/callback_tracing.h index 75b2fa16cc..56828397a6 100644 --- a/source/include/rocprofiler/callback_tracing.h +++ b/source/include/rocprofiler/callback_tracing.h @@ -28,9 +28,9 @@ ROCPROFILER_EXTERN_C_INIT -/** @defgroup CALLBACK_TRACING_SERVICE Synchronous Tracing Services - * - * Receive immediate callbacks on the calling thread +/** + * @defgroup CALLBACK_TRACING_SERVICE Synchronous Tracing Services + * @brief Receive immediate callbacks on the calling thread * * @{ */ diff --git a/source/include/rocprofiler/context.h b/source/include/rocprofiler/context.h index 34199ec455..9e5dfa4b16 100644 --- a/source/include/rocprofiler/context.h +++ b/source/include/rocprofiler/context.h @@ -28,7 +28,10 @@ ROCPROFILER_EXTERN_C_INIT /** - * @defgroup CONTEXT_OPERATIONS Context + * @defgroup CONTEXT_OPERATIONS Context Handling + * @brief Associate services with a handle. This handle is used to activate/deactivate the services + * during the application runtime. + * * @{ */ diff --git a/source/include/rocprofiler/counters.h b/source/include/rocprofiler/counters.h index cf762c4c64..d5e6d0d078 100644 --- a/source/include/rocprofiler/counters.h +++ b/source/include/rocprofiler/counters.h @@ -28,7 +28,9 @@ ROCPROFILER_EXTERN_C_INIT -/** @defgroup COUNTERS Hardware counters +/** + * @defgroup COUNTERS Hardware counters Information + * @brief Query functions related to hardware counters * @{ */ diff --git a/source/include/rocprofiler/defines.h b/source/include/rocprofiler/defines.h index bed0d10e97..2a1f4769dc 100644 --- a/source/include/rocprofiler/defines.h +++ b/source/include/rocprofiler/defines.h @@ -22,9 +22,10 @@ #pragma once -/** @defgroup SYMBOL_VERSIONING_GROUP Symbol Versions +/** + * @defgroup SYMBOL_VERSIONING_GROUP Symbol Versions * - * The names used for the shared library versioned symbols. + * @brief The names used for the shared library versioned symbols. * * Every function is annotated with one of the version macros defined in this * section. Each macro specifies a corresponding symbol version string. After diff --git a/source/include/rocprofiler/dispatch_profile.h b/source/include/rocprofiler/dispatch_profile.h index ddeaea755a..9f495624b9 100644 --- a/source/include/rocprofiler/dispatch_profile.h +++ b/source/include/rocprofiler/dispatch_profile.h @@ -30,8 +30,10 @@ ROCPROFILER_EXTERN_C_INIT -/** @defgroup DISPATCH_PROFILE_COUNTING_SERVICE Dispatch Profile Counting - * Service +/** + * @defgroup DISPATCH_PROFILE_COUNTING_SERVICE Dispatch Profile Counting Service + * @brief Per-dispatch hardware counter collection service + * * @{ */ diff --git a/source/include/rocprofiler/external_correlation.h b/source/include/rocprofiler/external_correlation.h index 85461e6611..bb41061d7d 100644 --- a/source/include/rocprofiler/external_correlation.h +++ b/source/include/rocprofiler/external_correlation.h @@ -29,8 +29,7 @@ ROCPROFILER_EXTERN_C_INIT /** * @defgroup EXTERNAL_CORRELATION External Correlation IDs - * - * User-defined correlation identifiers to supplement rocprofiler generated correlation ids + * @brief User-defined correlation identifiers to supplement rocprofiler generated correlation ids * * @{ */ diff --git a/source/include/rocprofiler/fwd.h b/source/include/rocprofiler/fwd.h index 45965da2ae..e5ab4c0816 100644 --- a/source/include/rocprofiler/fwd.h +++ b/source/include/rocprofiler/fwd.h @@ -37,8 +37,8 @@ ROCPROFILER_EXTERN_C_INIT /** * @defgroup BASIC_DATA_TYPES Basic data types + * @brief Basic data types and typedefs * - * Basic data types and typedefs * @{ */ @@ -290,8 +290,8 @@ typedef struct } rocprofiler_external_correlation_id_t; /** + * @struct rocprofiler_buffer_id_t * @brief Buffer ID. - * @addtogroup BUFFER_HANDLING */ typedef struct { diff --git a/source/include/rocprofiler/internal_threading.h b/source/include/rocprofiler/internal_threading.h index e8133c914e..8a5060b4b4 100644 --- a/source/include/rocprofiler/internal_threading.h +++ b/source/include/rocprofiler/internal_threading.h @@ -27,9 +27,9 @@ ROCPROFILER_EXTERN_C_INIT -/** @defgroup INTERNAL_THREADING Internal thread handling - * - * Callbacks before and after threads created internally by libraries +/** + * @defgroup INTERNAL_THREADING Internal Thread Handling + * @brief Callbacks before and after threads created internally by libraries * * @{ */ diff --git a/source/include/rocprofiler/pc_sampling.h b/source/include/rocprofiler/pc_sampling.h index 14992b6e45..bbf1c30d97 100644 --- a/source/include/rocprofiler/pc_sampling.h +++ b/source/include/rocprofiler/pc_sampling.h @@ -28,7 +28,9 @@ ROCPROFILER_EXTERN_C_INIT -/** @defgroup PC_SAMPLING_SERVICE PC Sampling Service +/** + * @defgroup PC_SAMPLING_SERVICE PC Sampling + * @brief Enabling PC (Program Counter) Sampling for GPU Activity * @{ */ diff --git a/source/include/rocprofiler/profile_config.h b/source/include/rocprofiler/profile_config.h index 21e07a7593..1ce14f9172 100644 --- a/source/include/rocprofiler/profile_config.h +++ b/source/include/rocprofiler/profile_config.h @@ -28,7 +28,9 @@ ROCPROFILER_EXTERN_C_INIT -/** @defgroup PROFILE_CONFIG Profile Configurations +/** + * @defgroup PROFILE_CONFIG Profile Configurations + * @brief Group one or more hardware counters into a unique handle * * @{ */ diff --git a/source/include/rocprofiler/rocprofiler.h b/source/include/rocprofiler/rocprofiler.h index f3c466fdd7..6941aa70da 100644 --- a/source/include/rocprofiler/rocprofiler.h +++ b/source/include/rocprofiler/rocprofiler.h @@ -26,18 +26,44 @@ #include #include "rocprofiler/defines.h" +#include "rocprofiler/fwd.h" -/** @defgroup VERSIONING_GROUP Library Versioning - * - * Version information about the interface and the associated installed - * library. +/** + * @defgroup VERSIONING_GROUP Library Versioning + * @brief Version information about the interface and the associated installed library. * * The semantic version of the interface following semver.org rules. A context * that uses this interface is only compatible with the installed library if * the major version numbers match and the interface minor version number is * less than or equal to the installed library minor version number. + * + * @{ */ +#include "rocprofiler/version.h" + +ROCPROFILER_EXTERN_C_INIT + +/** + * @fn rocprofiler_status_t rocprofiler_get_version(uint32_t* major, uint32_t* minor, uint32_t* + * patch) + * @brief Query the version of the installed library. + * + * Return the version of the installed library. This can be used to check if + * it is compatible with this interface version. This function can be used + * even when the library is not initialized. + * + * @param [out] major The major version number is stored if non-NULL. + * @param [out] minor The minor version number is stored if non-NULL. + * @param [out] patch The patch version number is stored if non-NULL. + */ +rocprofiler_status_t +rocprofiler_get_version(uint32_t* major, uint32_t* minor, uint32_t* patch) ROCPROFILER_API; + +ROCPROFILER_EXTERN_C_FINI + +/** @} */ + #include "rocprofiler/agent.h" #include "rocprofiler/agent_profile.h" #include "rocprofiler/buffer.h" @@ -47,42 +73,29 @@ #include "rocprofiler/counters.h" #include "rocprofiler/dispatch_profile.h" #include "rocprofiler/external_correlation.h" -#include "rocprofiler/fwd.h" #include "rocprofiler/hip.h" #include "rocprofiler/hsa.h" #include "rocprofiler/marker.h" #include "rocprofiler/pc_sampling.h" #include "rocprofiler/profile_config.h" #include "rocprofiler/spm.h" -#include "rocprofiler/version.h" ROCPROFILER_EXTERN_C_INIT /** - * @brief Query the version of the installed library. + * @defgroup MISCELLANEOUS_GROUP Miscellaneous Utility Functions * - * Return the version of the installed library. This can be used to check if - * it is compatible with this interface version. This function can be used - * even when the library is not initialized. - * - * @param [out] major The major version number is stored if non-NULL. - * @param [out] minor The minor version number is stored if non-NULL. - * @param [out] patch The patch version number is stored if non-NULL. - * @addtogroup VERSIONING_GROUP - */ -rocprofiler_status_t -rocprofiler_get_version(uint32_t* major, uint32_t* minor, uint32_t* patch) ROCPROFILER_API; - -/** - * @defgroup MISCELLANEOUS_GROUP Miscellaneous utility functions + * @{ */ /** + * @fn rocprofiler_status_t rocprofiler_get_timestamp(rocprofiler_timestamp_t* ts) * @brief Get the timestamp value that rocprofiler uses * @param [out] ts Output address of the rocprofiler timestamp value - * @addtogroup MISCELLANEOUS_GROUP */ rocprofiler_status_t rocprofiler_get_timestamp(rocprofiler_timestamp_t* ts) ROCPROFILER_API ROCPROFILER_NONNULL(1); +/** @} */ + ROCPROFILER_EXTERN_C_FINI diff --git a/source/include/rocprofiler/rocprofiler_plugin.h b/source/include/rocprofiler/rocprofiler_plugin.h index 209d136a2c..41ac0a60b3 100644 --- a/source/include/rocprofiler/rocprofiler_plugin.h +++ b/source/include/rocprofiler/rocprofiler_plugin.h @@ -47,45 +47,45 @@ #include -ROCPROFILER_EXTERN_C_INIT /* __cplusplus */ - - /** @defgroup rocprofiler_plugins ROCProfiler Plugin API Specification - * @{ - */ - - /** @defgroup initialization_group Initialization and Finalization - * @ingroup rocprofiler_plugins - * - * The ROCProfiler Plugin API must be initialized before using any of the - * operations to report trace data, and finalized after the last trace data has - * been reported. - * - * @{ - */ - - /** - * Initialize plugin. - * Must be called before any other operation. - * - * @param[in] rocprofiler_major_version The major version of the ROCProfiler API - * being used by the ROCProfiler Tool. An error is reported if this does not - * match the major version of the ROCProfiler API used to build the plugin - * library. This ensures compatibility of the trace data format. - * @param[in] rocprofiler_minor_version The minor version of the ROCProfiler API - * being used by the ROCProfiler Tool. An error is reported if the - * @p rocprofiler_major_version matches and this is greater than the minor - * version of the ROCProfiler API used to build the plugin library. This ensures - * compatibility of the trace data format. - * @param[in] data Pointer to the data passed to the ROCProfiler Plugin by the tool - * @return Returns 0 on success and -1 on error. - */ - ROCPROFILER_EXPORT int - rocprofiler_plugin_initialize(uint32_t rocprofiler_major_version, - uint32_t rocprofiler_minor_version, - void* data); +ROCPROFILER_EXTERN_C_INIT /** - * Finalize plugin. + * @defgroup ROCPROFILER_PLUGINS ROCProfiler Plugin API Specification + * @{ + */ + +/** + * @defgroup INITIALIZATION_GROUP Initialization and Finalization + * @brief The ROCProfiler Plugin API must be initialized before using any of the + * operations to report trace data, and finalized after the last trace data has + * been reported. + * @ingroup ROCPROFILER_PLUGINS + * + * @{ + */ + +/** + * @brief Initialize plugin. Must be called before any other operation. + * + * @param[in] rocprofiler_major_version The major version of the ROCProfiler API + * being used by the ROCProfiler Tool. An error is reported if this does not + * match the major version of the ROCProfiler API used to build the plugin + * library. This ensures compatibility of the trace data format. + * @param[in] rocprofiler_minor_version The minor version of the ROCProfiler API + * being used by the ROCProfiler Tool. An error is reported if the + * @p rocprofiler_major_version matches and this is greater than the minor + * version of the ROCProfiler API used to build the plugin library. This ensures + * compatibility of the trace data format. + * @param[in] data Pointer to the data passed to the ROCProfiler Plugin by the tool + * @return Returns 0 on success and -1 on error. + */ +ROCPROFILER_EXPORT int +rocprofiler_plugin_initialize(uint32_t rocprofiler_major_version, + uint32_t rocprofiler_minor_version, + void* data); + +/** + * @brief Finalize plugin. * This must be called after ::rocprofiler_plugin_initialize and after all * profiling data has been reported by * rocprofiler_plugin_write_kernel_records @@ -95,9 +95,11 @@ rocprofiler_plugin_finalize(); /** @} */ -/** @defgroup profiling_record_write_functions Profiling data reporting - * @ingroup rocprofiler_plugins - * Operations to output profiling data. +/** + * @defgroup profiling_record_write_functions Profiling data reporting + * @brief Operations to output profiling data. + * @ingroup ROCPROFILER_PLUGINS + * * @{ */ @@ -119,7 +121,7 @@ rocprofiler_plugin_write_buffer_records(rocprofiler_context_id_t context_id size_t num_headers); /** - * Report Synchronous Record. + * @brief Report Synchronous Record. * * @param[in] record Synchronous Tracer record. * @return Returns 0 on success and -1 on error. @@ -129,7 +131,6 @@ ROCPROFILER_EXPORT int rocprofiler_plugin_write_record(rocprofiler_record_header_t record); /** @} */ - /** @} */ ROCPROFILER_EXTERN_C_FINI diff --git a/source/include/rocprofiler/spm.h b/source/include/rocprofiler/spm.h index c23bcf98d7..420e1a19ff 100644 --- a/source/include/rocprofiler/spm.h +++ b/source/include/rocprofiler/spm.h @@ -27,7 +27,10 @@ ROCPROFILER_EXTERN_C_INIT -/** @defgroup SPM_SERVICE SPM Service +/** + * @defgroup SPM_SERVICE SPM Service + * @brief Streaming Performance Monitoring + * * @{ */ diff --git a/source/scripts/setup-sanitizer-env.sh b/source/scripts/setup-sanitizer-env.sh new file mode 100755 index 0000000000..7df9811542 --- /dev/null +++ b/source/scripts/setup-sanitizer-env.sh @@ -0,0 +1,38 @@ +#!/bin/bash -e + +# +# This file will export the same environment variables for running sanitizers as run-ci.py +# This file is useful to set the suppressions files +# +# Example usage: +# +# source ./source/scripts/setup-sanitizer-env.sh +# + +SUPPR_DIR=$(cd $(dirname ${BASH_SOURCE[0]}) &> /dev/null && pwd) + +for i in $(seq 20 -1 8) +do + set +e + SYMBOLIZER=$(which llvm-symbolizer-${i}) + set -e + if [ -n "${SYMBOLIZER}" ]; then + : ${EXTERNAL_SYMBOLIZER_PATH:="${SYMBOLIZER}"} + fi +done + +if [ -n "${EXTERNAL_SYMBOLIZER_PATH}" ]; then + EXTERNAL_SYMBOLIZER=" external_symbolizer_path=${EXTERNAL_SYMBOLIZER_PATH}" +fi + +: ${ASAN_OPTIONS="detect_leaks=0 use_sigaltstack=0 suppressions=${SUPPR_DIR}/address-sanitizer-suppr.txt"} +: ${LSAN_OPTIONS="suppressions=${SUPPR_DIR}/leak-sanitizer-suppr.txt"} +: ${TSAN_OPTIONS="history_size=5 second_deadlock_stack=1 suppressions=${SUPPR_DIR}/thread-sanitizer-suppr.txt${EXTERNAL_SYMBOLIZER}"} + +export ASAN_OPTIONS +export LSAN_OPTIONS +export TSAN_OPTIONS + +echo "ASAN_OPTIONS=\"${ASAN_OPTIONS}\"" +echo "LSAN_OPTIONS=\"${LSAN_OPTIONS}\"" +echo "TSAN_OPTIONS=\"${TSAN_OPTIONS}\"" diff --git a/source/scripts/thread-sanitizer-suppr.txt b/source/scripts/thread-sanitizer-suppr.txt index 069460aa20..98846ed4c6 100644 --- a/source/scripts/thread-sanitizer-suppr.txt +++ b/source/scripts/thread-sanitizer-suppr.txt @@ -5,6 +5,9 @@ # leaked thread thread:libhsa-runtime64.so +# data race in operator delete(void*) +race_top:libamdhip64.so + # unlock of an unlocked mutex (or by a wrong thread) mutex:librocm_smi64.so