diff --git a/projects/rocprofiler/CMakeLists.txt b/projects/rocprofiler/CMakeLists.txt index 8ed42e5269..79ae6cca43 100644 --- a/projects/rocprofiler/CMakeLists.txt +++ b/projects/rocprofiler/CMakeLists.txt @@ -366,7 +366,7 @@ if(${LIBRARY_TYPE} STREQUAL SHARED) set(CPACK_DEBIAN_FILE_NAME "DEB-DEFAULT") set(CPACK_DEBIAN_RUNTIME_PACKAGE_NAME "${PROJECT_NAME}") set(CPACK_DEBIAN_RUNTIME_PACKAGE_DEPENDS - "rocminfo, hsa-rocr, rocm-core, libsystemd-dev, libelf-dev, libnuma-dev, libpciaccess-dev, libxml2-dev" + "rocminfo, hsa-rocr, rocm-core, libelf-dev, libnuma-dev, libxml2-dev" ) set(CPACK_DEBIAN_DEV_PACKAGE_NAME "${PROJECT_NAME}-dev") set(CPACK_DEBIAN_DEV_PACKAGE_DEPENDS "${PROJECT_NAME}, hsa-rocr-dev, rocm-core") @@ -409,7 +409,7 @@ if(${LIBRARY_TYPE} STREQUAL SHARED) set(CPACK_RPM_FILE_NAME "RPM-DEFAULT") set(CPACK_RPM_RUNTIME_PACKAGE_NAME "${PROJECT_NAME}") set(CPACK_RPM_RUNTIME_PACKAGE_REQUIRES - "rocminfo, hsa-rocr, rocm-core, systemd-devel, libpciaccess-devel, libxml2-devel") + "rocminfo, hsa-rocr, rocm-core, libxml2-devel") set(CPACK_RPM_DEV_PACKAGE_NAME "${PROJECT_NAME}-devel") set(CPACK_RPM_DEV_PACKAGE_REQUIRES "${PROJECT_NAME}, hsa-rocr-devel, rocm-core") set(CPACK_RPM_DEV_PACKAGE_PROVIDES "${PROJECT_NAME}-dev") diff --git a/projects/rocprofiler/README.md b/projects/rocprofiler/README.md index 0e1fce32c8..060fc2e445 100644 --- a/projects/rocprofiler/README.md +++ b/projects/rocprofiler/README.md @@ -58,6 +58,7 @@ cd .../rocprofiler/build export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH # paths to ROC profiler and oher libraries export HSA_TOOLS_LIB=librocprofiler64.so.1 # ROC profiler library loaded by HSA runtime export ROCP_TOOL_LIB=test/librocprof-tool.so # tool library loaded by ROC profiler +export LIBPCI_TOOL_LIB=libpciaccess.so # PCI access library loaded by ROC profiler export ROCP_METRICS=metrics.xml # ROC profiler metrics config file export ROCP_INPUT=input.xml # input file for the tool library export ROCP_OUTPUT_DIR=./ # output directory for the tool library, for metrics results file 'results.txt' and trace files @@ -125,7 +126,7 @@ Note: ROCProfilerV2 is currently considered a beta version and is subject to cha - rocm-llvm-dev - makecache - Gtest Development Package (Ubuntu: libgtest-dev) -- libsystemd-dev, libelf-dev, libnuma-dev, libpciaccess-dev on ubuntu or their corresponding packages on any other OS +- libelf-dev, libnuma-dev, libpciaccess-dev on ubuntu or their corresponding packages on any other OS - Cppheaderparser, websockets, matplotlib, lxml, barectf Python3 Packages - Python packages can be installed using: diff --git a/projects/rocprofiler/doc/rocprofv2_tool.md b/projects/rocprofiler/doc/rocprofv2_tool.md index 52ae924a0a..84c6d184d8 100644 --- a/projects/rocprofiler/doc/rocprofv2_tool.md +++ b/projects/rocprofiler/doc/rocprofv2_tool.md @@ -32,7 +32,7 @@ The first API library version for ROCProfiler v2 is 9.0.0 - makecache - Gtest Development Package (Ubuntu: libgtest-dev) -- libsystemd-dev, libelf-dev, libnuma-dev on ubuntu or their corresponding packages on any other OS +- libelf-dev, libnuma-dev on ubuntu or their corresponding packages on any other OS - Cppheaderparser, websockets, matplotlib, lxml, barectf Python3 Packages ### Build diff --git a/projects/rocprofiler/samples/CMakeLists.txt b/projects/rocprofiler/samples/CMakeLists.txt index 6534a27841..7e39ed4921 100644 --- a/projects/rocprofiler/samples/CMakeLists.txt +++ b/projects/rocprofiler/samples/CMakeLists.txt @@ -127,7 +127,7 @@ rocprofiler_sample_add_executable( target_include_directories( pcie_counters_sampler PRIVATE ${PROJECT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/common) -target_link_libraries(pcie_counters_sampler PRIVATE rocprofiler-v2 systemd amd_comgr) +target_link_libraries(pcie_counters_sampler PRIVATE rocprofiler-v2 amd_comgr) target_link_options(pcie_counters_sampler PRIVATE "-Wl,--build-id=md5") add_dependencies(samples pcie_counters_sampler) install(TARGETS pcie_counters_sampler @@ -143,7 +143,7 @@ rocprofiler_sample_add_executable( target_include_directories( xgmi_counters_sampler PRIVATE ${PROJECT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/common) -target_link_libraries(xgmi_counters_sampler PRIVATE rocprofiler-v2 systemd amd_comgr) +target_link_libraries(xgmi_counters_sampler PRIVATE rocprofiler-v2 amd_comgr) target_link_options(xgmi_counters_sampler PRIVATE "-Wl,--build-id=md5") add_dependencies(samples xgmi_counters_sampler) install(TARGETS xgmi_counters_sampler diff --git a/projects/rocprofiler/src/api/CMakeLists.txt b/projects/rocprofiler/src/api/CMakeLists.txt index 61782f088d..0896b5bf8d 100644 --- a/projects/rocprofiler/src/api/CMakeLists.txt +++ b/projects/rocprofiler/src/api/CMakeLists.txt @@ -337,7 +337,6 @@ target_link_libraries( amd_comgr dw elf - ${PCIACCESS_LIBRARIES} rocprofiler::memcheck) get_target_property(ROCPROFILER_LIBRARY_V2_NAME rocprofiler-v2 OUTPUT_NAME) diff --git a/projects/rocprofiler/src/core/counters/mmio/mmio.cpp b/projects/rocprofiler/src/core/counters/mmio/mmio.cpp index c19db4f587..6adb49597f 100644 --- a/projects/rocprofiler/src/core/counters/mmio/mmio.cpp +++ b/projects/rocprofiler/src/core/counters/mmio/mmio.cpp @@ -21,8 +21,10 @@ #include "mmio.h" #include #include "../../../utils/helper.h" +#include "../../../utils/libpci_helper.h" #include "pcie_perfmon_registers_mi200.h" #include "df_perfmon_registers_mi200.h" + namespace rocprofiler { namespace mmio { @@ -57,23 +59,24 @@ MMIO::MMIO(const HSAAgentInfo& info) const auto pci_location_id = agent_info_->GetDeviceInfo().getPCILocationID(); pci_device_ = - pci_device_find_by_slot(pci_domain, pci_location_id >> 8, pci_location_id & 0xFF, 0); - if (!pci_device_ || pci_device_probe(pci_device_)) fatal("failed to probe the GPU device\n"); + GetPciAccessLibApi()->pci_device_find_by_slot(pci_domain, pci_location_id >> 8, pci_location_id & 0xFF, 0); + if (!pci_device_ || GetPciAccessLibApi()->pci_device_probe(pci_device_)) fatal("failed to probe the GPU device\n"); // Look for a region between 256KB and 4096KB, 32-bit, non IO, and non prefetchable. for (size_t region = 0; region < sizeof(pci_device::regions) / sizeof(pci_device::regions[0]); - ++region) + ++region){ if (pci_device_->regions[region].is_64 == 0 && pci_device_->regions[region].is_prefetchable == 0 && pci_device_->regions[region].is_IO == 0 && pci_device_->regions[region].size >= (256UL * 1024) && pci_device_->regions[region].size <= (4096UL * 1024)) { pci_memory_size_ = pci_device_->regions[region].size; - int err = pci_device_map_range(pci_device_, pci_device_->regions[region].base_addr, + int err = GetPciAccessLibApi()->pci_device_map_range(pci_device_, pci_device_->regions[region].base_addr, pci_device_->regions[region].size, PCI_DEV_MAP_FLAG_WRITABLE, (void**)&pci_memory_); if (err) fatal("failed to map the registers. Error code: %d\n", err); } + } if (pci_memory_ == nullptr) fatal("could not find the pci memory address\n"); @@ -82,7 +85,7 @@ MMIO::MMIO(const HSAAgentInfo& info) MMIO::~MMIO() { if (pci_memory_) { - int err = pci_device_unmap_range(pci_device_, pci_memory_, pci_memory_size_); + int err = GetPciAccessLibApi()->pci_device_unmap_range(pci_device_, pci_memory_, pci_memory_size_); if (err) warning("failed to unmap the pci memory. Error code: %d\n", err); } } diff --git a/projects/rocprofiler/src/core/session/counters_sampler.cpp b/projects/rocprofiler/src/core/session/counters_sampler.cpp index 870662a009..2b5462c400 100644 --- a/projects/rocprofiler/src/core/session/counters_sampler.cpp +++ b/projects/rocprofiler/src/core/session/counters_sampler.cpp @@ -23,6 +23,7 @@ #include "src/api/rocprofiler_singleton.h" #include "src/core/counters/mmio/pcie_counters_mi200.h" #include "src/core/counters/mmio/df_counters_mi200.h" +#include "src/utils/libpci_helper.h" namespace rocprofiler { @@ -32,9 +33,9 @@ CountersSampler::CountersSampler(rocprofiler_buffer_id_t buffer_id, : buffer_id_(buffer_id), filter_id_(filter_id), session_id_(session_id), - pci_system_initialized_(pci_system_init() == 0) - + pci_system_initialized_(false) { + pci_system_initialized_ = GetPciAccessLibApi()->pci_system_init() == 0; params_ = rocprofiler::ROCProfiler_Singleton::GetInstance() .GetSession(session_id_) ->GetFilter(filter_id_) @@ -94,8 +95,9 @@ CountersSampler::~CountersSampler() { // clean up libpcieaccess resources // TODO: should be part of mmio class in future if (pci_system_initialized_) { - pci_system_cleanup(); + GetPciAccessLibApi()->pci_system_cleanup(); pci_system_initialized_ = false; + UnLoadPcieAccessLibAPI(); } } diff --git a/projects/rocprofiler/src/pcsampler/core/pc_sampler.cpp b/projects/rocprofiler/src/pcsampler/core/pc_sampler.cpp index 760322462f..5a9e0ae188 100644 --- a/projects/rocprofiler/src/pcsampler/core/pc_sampler.cpp +++ b/projects/rocprofiler/src/pcsampler/core/pc_sampler.cpp @@ -25,12 +25,12 @@ #include #include -#include #include "src/api/rocprofiler_singleton.h" #include "src/pcsampler/session/pc_sampler.h" #include "src/pcsampler/gfxip/gfxip.h" #include "src/core/hsa/hsa_support.h" +#include "src/utils/libpci_helper.h" namespace rocprofiler::pc_sampler { @@ -39,12 +39,16 @@ PCSampler::PCSampler(rocprofiler_buffer_id_t buffer_id, rocprofiler_filter_id_t : buffer_id_(buffer_id), filter_id_(filter_id), session_id_(session_id), - pci_system_initialized_(pci_system_init() == 0) {} + pci_system_initialized_(false) { + pci_system_initialized_ = GetPciAccessLibApi()->pci_system_init() == 0; +} PCSampler::~PCSampler() { if (pci_system_initialized_) { - pci_system_cleanup(); + GetPciAccessLibApi()->pci_system_cleanup(); + pci_system_initialized_ = false; + UnLoadPcieAccessLibAPI(); } } diff --git a/projects/rocprofiler/src/pcsampler/gfxip/gfxip.cpp b/projects/rocprofiler/src/pcsampler/gfxip/gfxip.cpp index 5899f0c7ea..a7c9ac7828 100644 --- a/projects/rocprofiler/src/pcsampler/gfxip/gfxip.cpp +++ b/projects/rocprofiler/src/pcsampler/gfxip/gfxip.cpp @@ -33,6 +33,7 @@ #include "gfxip.h" #include "src/utils/helper.h" +#include "src/utils/libpci_helper.h" static const char DEBUG_DRI_PATH[] = "/sys/kernel/debug/dri/"; static const char DEV_PFX[] = "dev="; @@ -142,6 +143,7 @@ device_t::device_t(const bool pci_inited, const HSAAgentInfo& info) }()); int instance = find_pci_instance(name); + bool use_pciaccess_library = false; { std::string dri_debug_path_pfx(DEBUG_DRI_PATH); dri_debug_path_pfx += std::to_string(instance); @@ -156,30 +158,33 @@ device_t::device_t(const bool pci_inited, const HSAAgentInfo& info) fatal(msg); } } else { - goto device_specific_init; + use_pciaccess_library = true; } } + if (use_pciaccess_library) { - pci_device_ = - pci_device_find_by_slot(pci_domain, pci_location_id >> 8, pci_location_id & 0xFF, 0); - if (!pci_device_ || pci_device_probe(pci_device_)) fatal("failed to probe the GPU device\n"); + pci_device_ = + GetPciAccessLibApi()->pci_device_find_by_slot(pci_domain, pci_location_id >> 8, pci_location_id & 0xFF, 0); + if (!pci_device_ || GetPciAccessLibApi()->pci_device_probe(pci_device_)) fatal("failed to probe the GPU device\n"); - // Look for a region between 256KB and 4096KB, 32-bit, non IO, and non prefetchable. - for (size_t region = 0; region < sizeof(pci_device::regions) / sizeof(pci_device::regions[0]); - ++region) - if (pci_device_->regions[region].is_64 == 0 && - pci_device_->regions[region].is_prefetchable == 0 && - pci_device_->regions[region].is_IO == 0 && - pci_device_->regions[region].size >= (256UL * 1024) && - pci_device_->regions[region].size <= (4096UL * 1024)) { - pci_memory_size_ = pci_device_->regions[region].size; - if (pci_device_map_range(pci_device_, pci_device_->regions[region].base_addr, - pci_device_->regions[region].size, PCI_DEV_MAP_FLAG_WRITABLE, - (void**)&pci_memory_)) - fatal("failed to map the registers\n"); + // Look for a region between 256KB and 4096KB, 32-bit, non IO, and non prefetchable. + for (size_t region = 0; region < sizeof(pci_device::regions) / sizeof(pci_device::regions[0]); + ++region) { + if (pci_device_->regions[region].is_64 == 0 && + pci_device_->regions[region].is_prefetchable == 0 && + pci_device_->regions[region].is_IO == 0 && + pci_device_->regions[region].size >= (256UL * 1024) && + pci_device_->regions[region].size <= (4096UL * 1024)) { + pci_memory_size_ = pci_device_->regions[region].size; + if (GetPciAccessLibApi()->pci_device_map_range(pci_device_, pci_device_->regions[region].base_addr, + pci_device_->regions[region].size, PCI_DEV_MAP_FLAG_WRITABLE, + (void**)&pci_memory_)) + fatal("failed to map the registers\n"); + } } - if (pci_memory_ == nullptr) fatal("could not find the pci memory address\n"); + if (pci_memory_ == nullptr) fatal("could not find the pci memory address\n"); + } device_specific_init: // FIXME: detect the gfxip and call the correct routine. @@ -187,7 +192,7 @@ device_specific_init: } device_t::~device_t() { - if (pci_memory_ && pci_device_unmap_range(pci_device_, pci_memory_, pci_memory_size_)) { + if (pci_memory_ && GetPciAccessLibApi()->pci_device_unmap_range(pci_device_, pci_memory_, pci_memory_size_)) { warning("failed to unmap the pci memory\n"); } } diff --git a/projects/rocprofiler/src/utils/libpci_helper.cpp b/projects/rocprofiler/src/utils/libpci_helper.cpp new file mode 100644 index 0000000000..1656e36056 --- /dev/null +++ b/projects/rocprofiler/src/utils/libpci_helper.cpp @@ -0,0 +1,134 @@ +/* Copyright (c) 2022 Advanced Micro Devices, Inc. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. */ + +#include "libpci_helper.h" + +#include +#include +#include "exception.h" + + +namespace rocprofiler { +PcieAccessApi* api; +void* libpciaccess_handle; + +// A function just to load all the symbols you need +PcieAccessApi* LoadPcieAccessLibAPI() { + char* libpciaccess_tool = getenv("LIBPCI_TOOL_LIB"); + if (libpciaccess_tool == nullptr) { + libpciaccess_tool = "libpciaccess.so"; + } + libpciaccess_handle = dlopen(libpciaccess_tool, RTLD_NOW); + if (libpciaccess_handle == nullptr) { + fprintf(stderr, "ROCProfiler: can't load libpciaccess library \"%s\"\n", libpciaccess_tool); + fprintf(stderr, "%s\n", dlerror()); + abort(); + } + dlerror(); /* Clear any existing error */ + + api = new PcieAccessApi(); + api->pci_device_find_by_slot = + reinterpret_cast(dlsym( + libpciaccess_handle, "pci_device_find_by_slot")); + if (api->pci_device_find_by_slot == nullptr) { + fprintf(stderr, + "ROCProfiler: libpciaccess library corrupted, pci_device_find_by_slot() method is " + "expected\n"); + fprintf(stderr, "%s\n", dlerror()); + abort(); + } + api->pci_device_probe = + reinterpret_cast(dlsym( + libpciaccess_handle, "pci_device_probe")); + if (api->pci_device_probe == nullptr) { + fprintf(stderr, + "ROCProfiler: libpciaccess library corrupted, pci_device_probe() method is " + "expected\n"); + fprintf(stderr, "%s\n", dlerror()); + abort(); + } + api->pci_device_map_range = + reinterpret_cast(dlsym( + libpciaccess_handle, "pci_device_map_range")); + if (api->pci_device_map_range == nullptr) { + fprintf(stderr, + "ROCProfiler: libpciaccess library corrupted, pci_device_map_range() method is " + "expected\n"); + fprintf(stderr, "%s\n", dlerror()); + abort(); + } + api->pci_device_unmap_range = + reinterpret_cast(dlsym( + libpciaccess_handle, "pci_device_unmap_range")); + + if (api->pci_device_unmap_range == nullptr) { + fprintf(stderr, + "ROCProfiler: libpciaccess library corrupted, pci_device_unmap_range() method is " + "expected\n"); + fprintf(stderr, "%s\n", dlerror()); + abort(); + } + api->pci_system_init = + reinterpret_cast(dlsym( + libpciaccess_handle, "pci_system_init")); + if (api->pci_system_init == nullptr) { + fprintf(stderr, + "ROCProfiler: libpciaccess library corrupted, pci_system_init() method is " + "expected\n"); + fprintf(stderr, "%s\n", dlerror()); + abort(); + } + api->pci_system_cleanup = + reinterpret_cast(dlsym( + libpciaccess_handle, "pci_system_cleanup")); + if (api->pci_system_cleanup == nullptr) { + fprintf(stderr, + "ROCProfiler: libpciaccess library corrupted, pci_system_cleanup() method is " + "expected\n"); + fprintf(stderr, "%s\n", dlerror()); + abort(); + } + return api; +} + +const PcieAccessApi* GetPciAccessLibApi() { + if (api == nullptr) { + api = LoadPcieAccessLibAPI(); + } + return api; +} + +void UnLoadPcieAccessLibAPI() { + if (api == nullptr) { + return; + } + api->pci_device_find_by_slot = nullptr; + api->pci_device_probe = nullptr; + api->pci_device_map_range = nullptr; + api->pci_device_unmap_range = nullptr; + api->pci_system_init = nullptr; + api->pci_system_cleanup = nullptr; + delete api; + api = nullptr; + if (dlclose(libpciaccess_handle) != 0) { + warning("Warning: Failed to close libpciaccess handle: %s\n", dlerror()); + } +} +} // namespace rocprofiler diff --git a/projects/rocprofiler/src/utils/libpci_helper.h b/projects/rocprofiler/src/utils/libpci_helper.h new file mode 100644 index 0000000000..e42144da87 --- /dev/null +++ b/projects/rocprofiler/src/utils/libpci_helper.h @@ -0,0 +1,51 @@ +/* Copyright (c) 2022 Advanced Micro Devices, Inc. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. */ +#pragma once + +#include + +namespace rocprofiler { + typedef struct pci_device * (*pci_device_find_by_slot_handler_t)(uint32_t, + uint32_t, uint32_t, uint32_t); + typedef int (*pci_device_probe_handler_t)(struct pci_device *); + typedef int (*pci_device_map_range_handler_t)(struct pci_device *, pciaddr_t, + pciaddr_t, unsigned, void **); + typedef int (*pci_device_unmap_range_handler_t) (struct pci_device *, void *, + pciaddr_t); + typedef int (*pci_system_init_handler_t)(); + typedef void (*pci_system_cleanup_handler_t)(); + + struct PcieAccessApi{ + pci_device_find_by_slot_handler_t pci_device_find_by_slot; + pci_device_probe_handler_t pci_device_probe; + pci_device_map_range_handler_t pci_device_map_range; + pci_device_unmap_range_handler_t pci_device_unmap_range; + pci_system_init_handler_t pci_system_init; + pci_system_cleanup_handler_t pci_system_cleanup; + }; + + + // A function just to load all the libpciaccess symbols + PcieAccessApi* LoadPcieAccessLibAPI(); + // A simple getter function for easy access to the API table + const PcieAccessApi* GetPciAccessLibApi(); + // Sets all API table functions to null and calls dlclose + void UnLoadPcieAccessLibAPI(); +} // namespace rocprofiler