[rocprofv3] SQLite3 database output (rocpd) support + rocprofiler-sdk-rocpd (#403)

* [rocprofv3] rocpd SQLite3 database output support

* Move counters xml and yaml to source/share/rocprofiler-sdk

- more representative of install hierarchy

* Add share/rocprofiler-sdk/rocpd SQL files

* Experimental rocprofiler-sdk SQL API

* rocprofv3 default output format is rocpd

* Fix rocpd event ids for counter collection w/o kernel dispatch

* Remove fktable entries from rocpd_tables.sql

* Fix rocpd schema path

* Fix install component for roctx python bindings

* rocprofiler-sdk-rocpd

- create include/rocprofiler-sdk-rocpd
- create rocprofiler-sdk-rocpd library, package, etc.
- default all "guid" fields to "{{guid}}" in tables
- remove "{{view_uuid}}" support (always unused)

* Migrate rocprofv3 to use rocprofiler-sdk-rocpd

* Fix missing foreign key reference

* Revert change

* Fix cmake comment

* Fix maybe-uninitialized compiler warning

* Fix maybe-uninitialized compiler warning

* Add logging to rocpd_sql_load_schema

* Improve string sanitization when inserting json strings

* Initialize rocpd logging on rocprofiler-sdk-rocpd library load

* Revert lib/output/generatePerfetto.cpp changes

* [temporary] Tweak rocprofv3-test-list-avail-trace-execute test log level

* Update get_install_path for lib/rocprofiler-sdk-rocpd/sql.cpp

- try to resolve issues on RHEL/SLES for dladdr

* Update lib/common/logging.cpp

- enable environ overrides

* dlsym for rocpd_sql_load_schema

* Make dl_info.dli_fname lexically normal

* Implement node_info alternatives if /etc/machine-id does not exist

* Misc include fixes

* SHA256 and UUIDv7 support

* Implement UUIDv7 in generateRocpd.cpp

* Support push/pop environment variables

* Minor tweak

* Fix glog segfaults when unsetting glog env

* Updated CHANGELOG

* Updates tests/pytest-packages

- rocpd_reader.py: RocpdReader

* Update tests / marker_views.sql

- add test_rocpd_data

* Update rocpd_tables.sql

- Use AUTOINCREMENT
- insert "uuid" and "guid" into rocpd_metadata

* Minor updates to generateRocpd.cpp

- don't quote GUID
- use sqlite3_open_v2
- use sqlite3_close_v2

* Update execute_raw_sql_statements_impl

- uses sqlite3_last_insert_rowid for autoincrement

* Update SQL deferred_transaction

- CI check for nullptr to connection

* Apply suggestions from code review

Co-authored-by: Welton, Benjamin <Benjamin.Welton@amd.com>

* Code review updates

- formatting
- replace if with switch
- remove loop for {{uuid}}

* Fix pmc_groups handling in rocprofv3

* Address code review feedback

- Include rocm_version in rocprofv3 version info
- Note `--version` option for `rocprofv3` in CHANGELOG.md
- remove commented out code

* Fix packaging dependencies

* Fix install package step of CI workflow

* Fix install package step of CI workflow

---------

Co-authored-by: Jonathan R. Madsen <jonathanrmadsen@gmail.com>
Co-authored-by: Welton, Benjamin <Benjamin.Welton@amd.com>

cmake/Templates/rocprofiler-sdk-rocpd/build-config.cmake.in

@@ -0,0 +1,9 @@
+# Config file for @PACKAGE_NAME@ and its component libraries in the build tree
+#
+
+list(APPEND @PACKAGE_NAME@_INCLUDE_DIR @CMAKE_BINARY_DIR@/source/include)
+
+foreach(COMP @PROJECT_BUILD_TREE_TARGETS@)
+    list(APPEND @PACKAGE_NAME@_LIBRARIES ${COMP})
+    target_link_libraries(@PACKAGE_NAME@::@PACKAGE_NAME@ INTERFACE ${COMP})
+endforeach()

cmake/Templates/rocprofiler-sdk-rocpd/config.cmake.in

@@ -0,0 +1,73 @@
+# - Config file for @PACKAGE_NAME@ and its component libraries
+# It defines the following variables:
+#
+# @PACKAGE_NAME@_VERSION
+# @PACKAGE_NAME@_INCLUDE_DIR
+# @PACKAGE_NAME@_LIB_DIR
+# @PACKAGE_NAME@_LIBRARIES
+#
+# It provides the following interface libraries:
+#
+# @PACKAGE_NAME@::@PACKAGE_NAME@
+#
+
+# prevent "target already exists" error
+include_guard(DIRECTORY)
+
+# compute paths
+get_filename_component(@PACKAGE_NAME@_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH)
+
+@PACKAGE_INIT@
+
+set_and_check(@PACKAGE_NAME@_INCLUDE_DIR "@PACKAGE_INCLUDE_INSTALL_DIR@")
+set_and_check(@PACKAGE_NAME@_LIB_DIR "@PACKAGE_LIB_INSTALL_DIR@")
+get_filename_component(@PACKAGE_NAME@_ROOT_DIR ${@PACKAGE_NAME@_INCLUDE_DIR} PATH)
+set_and_check(@PACKAGE_NAME@_ROOT "${@PACKAGE_NAME@_ROOT_DIR}")
+
+# extra validation
+foreach(_@PACKAGE_NAME@_SUBDIR @PROJECT_EXTRA_DIRS@)
+    set_and_check(_@PACKAGE_NAME@_SUBDIR_CHECK
+                  "${PACKAGE_PREFIX_DIR}/${_@PACKAGE_NAME@_SUBDIR}")
+    unset(_@PACKAGE_NAME@_SUBDIR_CHECK)
+endforeach()
+
+set(@PACKAGE_NAME@_LIBRARIES)
+
+# add interface library
+add_library(@PACKAGE_NAME@::@PACKAGE_NAME@ INTERFACE IMPORTED)
+
+if(@PACKAGE_NAME@_BUILD_TREE
+   AND EXISTS "${CMAKE_CURRENT_LIST_DIR}/@PACKAGE_NAME@-build-config.cmake")
+    include("${CMAKE_CURRENT_LIST_DIR}/@PACKAGE_NAME@-build-config.cmake")
+else()
+    include("${@PACKAGE_NAME@_CMAKE_DIR}/@PACKAGE_NAME@-targets.cmake")
+
+    # Library dependencies
+    if(@PACKAGE_NAME@_FIND_COMPONENTS)
+        foreach(COMP ${@PACKAGE_NAME@_FIND_COMPONENTS})
+            set(TARG @PACKAGE_NAME@::@PACKAGE_NAME@-${COMP})
+            if(TARGET ${TARG})
+                set(@PACKAGE_NAME@_${COMP}_FOUND 1)
+                list(APPEND @PACKAGE_NAME@_LIBRARIES ${TARG})
+                target_link_libraries(@PACKAGE_NAME@::@PACKAGE_NAME@ INTERFACE ${TARG})
+            else()
+                set(@PACKAGE_NAME@_${COMP}_FOUND 0)
+            endif()
+        endforeach()
+    else()
+        foreach(TARG @PROJECT_BUILD_TARGETS@)
+            set(TARG @PACKAGE_NAME@::${TARG})
+            list(APPEND @PACKAGE_NAME@_LIBRARIES ${TARG})
+            target_link_libraries(@PACKAGE_NAME@::@PACKAGE_NAME@ INTERFACE ${TARG})
+        endforeach()
+    endif()
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(
+    @PACKAGE_NAME@
+    FOUND_VAR @PACKAGE_NAME@_FOUND
+    VERSION_VAR @PACKAGE_NAME@_VERSION
+    REQUIRED_VARS @PACKAGE_NAME@_ROOT_DIR @PACKAGE_NAME@_INCLUDE_DIR
+                  @PACKAGE_NAME@_LIB_DIR @PACKAGE_NAME@_LIBRARIES @PACKAGE_NAME@_VERSION
+    HANDLE_COMPONENTS HANDLE_VERSION_RANGE)

cmake/Templates/setup-env.sh.in

@@ -8,16 +8,35 @@ if [ ! -d "${BASEDIR}" ]; then
     return 1
 fi
 
+@PACKAGE_NAME_UNDERSCORED@_get_python3_path()
+{
+    local PYTHON3_EXECUTABLE=$(command -v python3 2> /dev/null)
+    if [ -n "${PYTHON3_EXECUTABLE}" ]; then
+        local PYTHON3_VERSION=$(${PYTHON3_EXECUTABLE} -c "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}')" 2> /dev/null)
+        local @PACKAGE_NAME_UNDERSCORED@_PYTHONPATH=${BASEDIR}/@CMAKE_INSTALL_LIBDIR@/python${PYTHON3_VERSION}/site-packages
+        if [ -d "${@PACKAGE_NAME_UNDERSCORED@_PYTHONPATH}" ]; then
+            echo ${BASEDIR}/@CMAKE_INSTALL_LIBDIR@/python${PYTHON3_VERSION}/site-packages
+        fi
+    fi
+}
+
 @PACKAGE_NAME_UNDERSCORED@_ROOT=${BASEDIR}
+@PACKAGE_NAME_UNDERSCORED@_DIR=${BASEDIR}/@CMAKE_INSTALL_LIBDIR@/cmake/@PACKAGE_NAME@
 PATH=${BASEDIR}/bin:${PATH}
 LD_LIBRARY_PATH=${BASEDIR}/@CMAKE_INSTALL_LIBDIR@:${LD_LIBRARY_PATH}
-PYTHONPATH=${BASEDIR}/@CMAKE_INSTALL_PYTHONDIR@:${PYTHONPATH}
 CMAKE_PREFIX_PATH=${BASEDIR}:${CMAKE_PREFIX_PATH}
-@PACKAGE_NAME_UNDERSCORED@_DIR=${BASEDIR}/@CMAKE_INSTALL_LIBDIR@/cmake/@PACKAGE_NAME@
+
+if [ -z "@CMAKE_INSTALL_PYTHONDIR@" ]; then
+    PYTHONPATH=$(@PACKAGE_NAME_UNDERSCORED@_get_python3_path):${PYTHONPATH}
+else
+    PYTHONPATH=${BASEDIR}/@CMAKE_INSTALL_PYTHONDIR@:${PYTHONPATH}
+fi
+
+unset @PACKAGE_NAME_UNDERSCORED@_get_python3_path
 
 export @PACKAGE_NAME_UNDERSCORED@_ROOT
+export @PACKAGE_NAME_UNDERSCORED@_DIR
 export PATH
 export LD_LIBRARY_PATH
-export PYTHONPATH
 export CMAKE_PREFIX_PATH
-export @PACKAGE_NAME_UNDERSCORED@_DIR
+export PYTHONPATH

cmake/rocprofiler_config_install_rocpd.cmake

@@ -0,0 +1,106 @@
+# include guard
+include_guard(GLOBAL)
+
+include(CMakePackageConfigHelpers)
+
+set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME rocpd)
+set(SDK_PACKAGE_NAME "${PROJECT_NAME}")
+set(PACKAGE_NAME "${PROJECT_NAME}-rocpd")
+
+install(
+    EXPORT ${PACKAGE_NAME}-targets
+    FILE ${PACKAGE_NAME}-targets.cmake
+    NAMESPACE ${PACKAGE_NAME}::
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PACKAGE_NAME}
+    COMPONENT rocpd)
+
+rocprofiler_install_env_setup_files(
+    NAME ${PACKAGE_NAME}
+    VERSION ${PROJECT_VERSION}
+    INSTALL_DIR ${CMAKE_INSTALL_DATAROOTDIR}
+    COMPONENT rocpd)
+
+# ------------------------------------------------------------------------------#
+# install tree
+#
+set(PROJECT_INSTALL_DIR ${CMAKE_INSTALL_PREFIX})
+set(INCLUDE_INSTALL_DIR ${CMAKE_INSTALL_INCLUDEDIR})
+set(LIB_INSTALL_DIR ${CMAKE_INSTALL_LIBDIR})
+set(PROJECT_BUILD_TARGETS ${PACKAGE_NAME}-shared-library)
+set(PROJECT_EXTRA_DIRS "${CMAKE_INSTALL_INCLUDEDIR}/${PACKAGE_NAME}")
+
+configure_package_config_file(
+    ${PROJECT_SOURCE_DIR}/cmake/Templates/${PACKAGE_NAME}/config.cmake.in
+    ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}/cmake/${PACKAGE_NAME}/${PACKAGE_NAME}-config.cmake
+    INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PACKAGE_NAME}
+    INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX}
+    PATH_VARS PROJECT_INSTALL_DIR INCLUDE_INSTALL_DIR LIB_INSTALL_DIR)
+
+write_basic_package_version_file(
+    ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}/cmake/${PACKAGE_NAME}/${PACKAGE_NAME}-config-version.cmake
+    VERSION ${PROJECT_VERSION}
+    COMPATIBILITY SameMinorVersion)
+
+install(
+    FILES
+        ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}/cmake/${PACKAGE_NAME}/${PACKAGE_NAME}-config.cmake
+        ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}/cmake/${PACKAGE_NAME}/${PACKAGE_NAME}-config-version.cmake
+    DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PACKAGE_NAME}
+    COMPONENT rocpd)
+
+export(PACKAGE ${PACKAGE_NAME})
+
+# ------------------------------------------------------------------------------#
+# build tree
+#
+set(${PACKAGE_NAME}_BUILD_TREE
+    ON
+    CACHE BOOL "" FORCE)
+
+set(PROJECT_BUILD_TREE_TARGETS ${SDK_PACKAGE_NAME}::${PACKAGE_NAME}-shared-library
+                               ${SDK_PACKAGE_NAME}::${SDK_PACKAGE_NAME}-stack-protector)
+
+configure_file(
+    ${PROJECT_SOURCE_DIR}/cmake/Templates/${PACKAGE_NAME}/build-config.cmake.in
+    ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}/cmake/${PACKAGE_NAME}/${PACKAGE_NAME}-build-config.cmake
+    @ONLY)
+
+file(RELATIVE_PATH rocp_bin2src_rel_path ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR})
+string(REPLACE "//" "/" rocp_inc_rel_path "${rocp_bin2src_rel_path}/source/include")
+
+set(_BUILDTREE_EXPORT_DIR
+    "${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}/cmake/${PACKAGE_NAME}")
+
+execute_process(
+    COMMAND ${CMAKE_COMMAND} -E create_symlink ${rocp_inc_rel_path}
+            ${PROJECT_BINARY_DIR}/include WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
+
+if(NOT EXISTS "${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}")
+    file(MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}")
+endif()
+
+if(NOT EXISTS "${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}/${PACKAGE_NAME}")
+    file(MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}/${PACKAGE_NAME}")
+endif()
+
+if(NOT EXISTS "${_BUILDTREE_EXPORT_DIR}")
+    file(MAKE_DIRECTORY "${_BUILDTREE_EXPORT_DIR}")
+endif()
+
+if(NOT EXISTS "${_BUILDTREE_EXPORT_DIR}/${PACKAGE_NAME}-targets.cmake")
+    file(TOUCH "${_BUILDTREE_EXPORT_DIR}/${PACKAGE_NAME}-targets.cmake")
+endif()
+
+export(
+    EXPORT ${PACKAGE_NAME}-targets
+    NAMESPACE ${PACKAGE_NAME}::
+    FILE "${_BUILDTREE_EXPORT_DIR}/${PACKAGE_NAME}-targets.cmake")
+
+set(${PACKAGE_NAME}_DIR
+    "${_BUILDTREE_EXPORT_DIR}"
+    CACHE PATH "${PACKAGE_NAME} build tree install" FORCE)
+
+install(
+    FILES ${PROJECT_SOURCE_DIR}/LICENSE
+    DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/doc/${PACKAGE_NAME}
+    COMPONENT rocpd)

cmake/rocprofiler_config_packaging.cmake

@@ -58,13 +58,14 @@ list(REMOVE_ITEM ROCPROFILER_PACKAGING_COMPONENTS "Development" "Unspecified")
 list(LENGTH ROCPROFILER_PACKAGING_COMPONENTS NUM_ROCPROFILER_PACKAGING_COMPONENTS)
 
 # the packages we will generate
-set(ROCPROFILER_COMPONENT_GROUPS "core" "docs" "tests" "roctx")
+set(ROCPROFILER_COMPONENT_GROUPS "core" "docs" "tests" "roctx" "rocpd")
 
 set(COMPONENT_GROUP_core_COMPONENTS "core" "development" "samples" "tools" "benchmark"
                                     "Development" "Unspecified")
 set(COMPONENT_GROUP_docs_COMPONENTS "docs")
 set(COMPONENT_GROUP_tests_COMPONENTS "tests")
 set(COMPONENT_GROUP_roctx_COMPONENTS "roctx")
+set(COMPONENT_GROUP_rocpd_COMPONENTS "rocpd")
 
 # variables for each component group. Note: eventually we will probably want to separate
 # the core to just be the runtime libraries, development to be the headers and cmake
@@ -74,18 +75,25 @@ set(COMPONENT_NAME_core "rocprofiler-sdk")
 set(COMPONENT_NAME_docs "rocprofiler-sdk-docs")
 set(COMPONENT_NAME_tests "rocprofiler-sdk-tests")
 set(COMPONENT_NAME_roctx "rocprofiler-sdk-roctx")
+set(COMPONENT_NAME_rocpd "rocprofiler-sdk-rocpd")
 
-set(COMPONENT_DEP_core "rocprofiler-sdk-roctx (>= ${PROJECT_VERSION})")
+set(COMPONENT_DEP_core "rocprofiler-sdk-roctx (>= ${PROJECT_VERSION})"
+                       "rocprofiler-sdk-rocpd (>= ${PROJECT_VERSION})")
 set(COMPONENT_DEP_docs "")
-set(COMPONENT_DEP_tests "rocprofiler-sdk (>= ${PROJECT_VERSION})")
-set(COMPONENT_DEP_roctx "")
+set(COMPONENT_DEP_tests
+    "rocprofiler-sdk (>= ${PROJECT_VERSION})"
+    "rocprofiler-sdk-roctx (>= ${PROJECT_VERSION})"
+    "rocprofiler-sdk-rocpd (>= ${PROJECT_VERSION})")
+set(COMPONENT_DEP_roctx "rocprofiler-register")
+set(COMPONENT_DEP_rocpd "")
 
 set(COMPONENT_DESC_core "rocprofiler-sdk libraries, headers, samples, and tools")
 set(COMPONENT_DESC_docs "rocprofiler-sdk documentation")
 set(COMPONENT_DESC_tests "rocprofiler-sdk tests")
 set(COMPONENT_DESC_roctx "ROCm Tools Extension library and headers")
+set(COMPONENT_DESC_rocpd "ROCm Profiling Data library and headers")
 
-set(EXPECTED_PACKAGING_COMPONENTS 6)
+set(EXPECTED_PACKAGING_COMPONENTS 7)
 if(ROCPROFILER_BUILD_DOCS)
     math(EXPR EXPECTED_PACKAGING_COMPONENTS "${EXPECTED_PACKAGING_COMPONENTS} + 1")
 endif()