SWDEV-469750: Load libpciaccess-dev functions with dlopen. Removed libsystemd-dev from CMakeLists.txt

Change-Id: I314c188fabc2dd4de3b633d722599bc11a046622 [ROCm/rocprofiler commit: 4ed346aa63]
2024-07-09 11:46:25 -05:00
parent 9f222d2fd0
commit 8d53eb2836
@@ -366,7 +366,7 @@ if(${LIBRARY_TYPE} STREQUAL SHARED)
    set(CPACK_DEBIAN_FILE_NAME "DEB-DEFAULT")
    set(CPACK_DEBIAN_RUNTIME_PACKAGE_NAME "${PROJECT_NAME}")
    set(CPACK_DEBIAN_RUNTIME_PACKAGE_DEPENDS
-        "rocminfo, hsa-rocr, rocm-core, libsystemd-dev, libelf-dev, libnuma-dev, libpciaccess-dev, libxml2-dev"
+        "rocminfo, hsa-rocr, rocm-core, libelf-dev, libnuma-dev, libxml2-dev"
        )
    set(CPACK_DEBIAN_DEV_PACKAGE_NAME "${PROJECT_NAME}-dev")
    set(CPACK_DEBIAN_DEV_PACKAGE_DEPENDS "${PROJECT_NAME}, hsa-rocr-dev, rocm-core")
@@ -409,7 +409,7 @@ if(${LIBRARY_TYPE} STREQUAL SHARED)
    set(CPACK_RPM_FILE_NAME "RPM-DEFAULT")
    set(CPACK_RPM_RUNTIME_PACKAGE_NAME "${PROJECT_NAME}")
    set(CPACK_RPM_RUNTIME_PACKAGE_REQUIRES
-        "rocminfo, hsa-rocr, rocm-core, systemd-devel, libpciaccess-devel, libxml2-devel")
+        "rocminfo, hsa-rocr, rocm-core, libxml2-devel")
    set(CPACK_RPM_DEV_PACKAGE_NAME "${PROJECT_NAME}-devel")
    set(CPACK_RPM_DEV_PACKAGE_REQUIRES "${PROJECT_NAME}, hsa-rocr-devel, rocm-core")
    set(CPACK_RPM_DEV_PACKAGE_PROVIDES "${PROJECT_NAME}-dev")
@@ -58,6 +58,7 @@ cd .../rocprofiler/build
 export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH # paths to ROC profiler and oher libraries
 export HSA_TOOLS_LIB=librocprofiler64.so.1 # ROC profiler library loaded by HSA runtime
 export ROCP_TOOL_LIB=test/librocprof-tool.so # tool library loaded by ROC profiler
+export LIBPCI_TOOL_LIB=libpciaccess.so # PCI access library loaded by ROC profiler
 export ROCP_METRICS=metrics.xml # ROC profiler metrics config file
 export ROCP_INPUT=input.xml # input file for the tool library
 export ROCP_OUTPUT_DIR=./ # output directory for the tool library, for metrics results file 'results.txt' and trace files
@@ -125,7 +126,7 @@ Note: ROCProfilerV2 is currently considered a beta version and is subject to cha
 - rocm-llvm-dev
 - makecache
 - Gtest Development Package (Ubuntu: libgtest-dev)
- libsystemd-dev, libelf-dev, libnuma-dev, libpciaccess-dev on ubuntu or their corresponding packages on any other OS
+- libelf-dev, libnuma-dev, libpciaccess-dev on ubuntu or their corresponding packages on any other OS
 - Cppheaderparser, websockets, matplotlib, lxml, barectf Python3 Packages
 - Python packages can be installed using:

@@ -32,7 +32,7 @@ The first API library version for ROCProfiler v2 is 9.0.0

 - makecache
 - Gtest Development Package (Ubuntu: libgtest-dev)
- libsystemd-dev, libelf-dev, libnuma-dev on ubuntu or their corresponding packages on any other OS
+- libelf-dev, libnuma-dev on ubuntu or their corresponding packages on any other OS
 - Cppheaderparser, websockets, matplotlib, lxml, barectf Python3 Packages

 ### Build
@@ -127,7 +127,7 @@ rocprofiler_sample_add_executable(
 target_include_directories(
    pcie_counters_sampler PRIVATE ${PROJECT_SOURCE_DIR}
                                  ${CMAKE_CURRENT_SOURCE_DIR}/common)
-target_link_libraries(pcie_counters_sampler PRIVATE rocprofiler-v2 systemd amd_comgr)
+target_link_libraries(pcie_counters_sampler PRIVATE rocprofiler-v2 amd_comgr)
 target_link_options(pcie_counters_sampler PRIVATE "-Wl,--build-id=md5")
 add_dependencies(samples pcie_counters_sampler)
 install(TARGETS pcie_counters_sampler
@@ -143,7 +143,7 @@ rocprofiler_sample_add_executable(
 target_include_directories(
    xgmi_counters_sampler PRIVATE ${PROJECT_SOURCE_DIR}
                                  ${CMAKE_CURRENT_SOURCE_DIR}/common)
-target_link_libraries(xgmi_counters_sampler PRIVATE rocprofiler-v2 systemd amd_comgr)
+target_link_libraries(xgmi_counters_sampler PRIVATE rocprofiler-v2 amd_comgr)
 target_link_options(xgmi_counters_sampler PRIVATE "-Wl,--build-id=md5")
 add_dependencies(samples xgmi_counters_sampler)
 install(TARGETS xgmi_counters_sampler
@@ -337,7 +337,6 @@ target_link_libraries(
            amd_comgr
            dw
            elf
-            ${PCIACCESS_LIBRARIES}
            rocprofiler::memcheck)

 get_target_property(ROCPROFILER_LIBRARY_V2_NAME rocprofiler-v2 OUTPUT_NAME)
@@ -21,8 +21,10 @@
 #include "mmio.h"
 #include <cstdint>
 #include "../../../utils/helper.h"
+#include "../../../utils/libpci_helper.h"
 #include "pcie_perfmon_registers_mi200.h"
 #include "df_perfmon_registers_mi200.h"
+
 namespace rocprofiler {

 namespace mmio {
@@ -57,23 +59,24 @@ MMIO::MMIO(const HSAAgentInfo& info)
  const auto pci_location_id = agent_info_->GetDeviceInfo().getPCILocationID();

  pci_device_ =
-      pci_device_find_by_slot(pci_domain, pci_location_id >> 8, pci_location_id & 0xFF, 0);
-  if (!pci_device_ || pci_device_probe(pci_device_)) fatal("failed to probe the GPU device\n");
+      GetPciAccessLibApi()->pci_device_find_by_slot(pci_domain, pci_location_id >> 8, pci_location_id & 0xFF, 0);
+  if (!pci_device_ || GetPciAccessLibApi()->pci_device_probe(pci_device_)) fatal("failed to probe the GPU device\n");

  // Look for a region between 256KB and 4096KB, 32-bit, non IO, and non prefetchable.
  for (size_t region = 0; region < sizeof(pci_device::regions) / sizeof(pci_device::regions[0]);
-       ++region)
+       ++region){
    if (pci_device_->regions[region].is_64 == 0 &&
        pci_device_->regions[region].is_prefetchable == 0 &&
        pci_device_->regions[region].is_IO == 0 &&
        pci_device_->regions[region].size >= (256UL * 1024) &&
        pci_device_->regions[region].size <= (4096UL * 1024)) {
      pci_memory_size_ = pci_device_->regions[region].size;
-      int err = pci_device_map_range(pci_device_, pci_device_->regions[region].base_addr,
+      int err = GetPciAccessLibApi()->pci_device_map_range(pci_device_, pci_device_->regions[region].base_addr,
                                     pci_device_->regions[region].size, PCI_DEV_MAP_FLAG_WRITABLE,
                                     (void**)&pci_memory_);
      if (err) fatal("failed to map the registers. Error code: %d\n", err);
    }
+  }

  if (pci_memory_ == nullptr) fatal("could not find the pci memory address\n");

@@ -82,7 +85,7 @@ MMIO::MMIO(const HSAAgentInfo& info)

 MMIO::~MMIO() {
  if (pci_memory_) {
-    int err = pci_device_unmap_range(pci_device_, pci_memory_, pci_memory_size_);
+    int err = GetPciAccessLibApi()->pci_device_unmap_range(pci_device_, pci_memory_, pci_memory_size_);
    if (err) warning("failed to unmap the pci memory. Error code: %d\n", err);
  }
 }
@@ -23,6 +23,7 @@
 #include "src/api/rocprofiler_singleton.h"
 #include "src/core/counters/mmio/pcie_counters_mi200.h"
 #include "src/core/counters/mmio/df_counters_mi200.h"
+#include "src/utils/libpci_helper.h"

 namespace rocprofiler {

@@ -32,9 +33,9 @@ CountersSampler::CountersSampler(rocprofiler_buffer_id_t buffer_id,
    : buffer_id_(buffer_id),
      filter_id_(filter_id),
      session_id_(session_id),
-      pci_system_initialized_(pci_system_init() == 0)
-
+      pci_system_initialized_(false)
 {
+  pci_system_initialized_ = GetPciAccessLibApi()->pci_system_init() == 0;
  params_ = rocprofiler::ROCProfiler_Singleton::GetInstance()
                .GetSession(session_id_)
                ->GetFilter(filter_id_)
@@ -94,8 +95,9 @@ CountersSampler::~CountersSampler() {
  // clean up libpcieaccess resources
  // TODO: should be part of mmio class in future
  if (pci_system_initialized_) {
-    pci_system_cleanup();
+    GetPciAccessLibApi()->pci_system_cleanup();
    pci_system_initialized_ = false;
+    UnLoadPcieAccessLibAPI();
  }
 }

@@ -25,12 +25,12 @@
 #include <unordered_map>

 #include <hsa/hsa.h>
-#include <pciaccess.h>

 #include "src/api/rocprofiler_singleton.h"
 #include "src/pcsampler/session/pc_sampler.h"
 #include "src/pcsampler/gfxip/gfxip.h"
 #include "src/core/hsa/hsa_support.h"
+#include "src/utils/libpci_helper.h"

 namespace rocprofiler::pc_sampler {

@@ -39,12 +39,16 @@ PCSampler::PCSampler(rocprofiler_buffer_id_t buffer_id, rocprofiler_filter_id_t
    : buffer_id_(buffer_id),
      filter_id_(filter_id),
      session_id_(session_id),
-      pci_system_initialized_(pci_system_init() == 0) {}
+      pci_system_initialized_(false) {
+  pci_system_initialized_ = GetPciAccessLibApi()->pci_system_init() == 0;
+}

 PCSampler::~PCSampler() {
  if (pci_system_initialized_) {
-    pci_system_cleanup();
+    GetPciAccessLibApi()->pci_system_cleanup();
+
    pci_system_initialized_ = false;
+    UnLoadPcieAccessLibAPI();
  }
 }

@@ -33,6 +33,7 @@

 #include "gfxip.h"
 #include "src/utils/helper.h"
+#include "src/utils/libpci_helper.h"

 static const char DEBUG_DRI_PATH[] = "/sys/kernel/debug/dri/";
 static const char DEV_PFX[] = "dev=";
@@ -142,6 +143,7 @@ device_t::device_t(const bool pci_inited, const HSAAgentInfo& info)
  }());

  int instance = find_pci_instance(name);
+  bool use_pciaccess_library = false;
  {
    std::string dri_debug_path_pfx(DEBUG_DRI_PATH);
    dri_debug_path_pfx += std::to_string(instance);
@@ -156,30 +158,33 @@ device_t::device_t(const bool pci_inited, const HSAAgentInfo& info)
        fatal(msg);
      }
    } else {
-      goto device_specific_init;
+      use_pciaccess_library = true;
    }
  }
+  if (use_pciaccess_library) {

-  pci_device_ =
-      pci_device_find_by_slot(pci_domain, pci_location_id >> 8, pci_location_id & 0xFF, 0);
-  if (!pci_device_ || pci_device_probe(pci_device_)) fatal("failed to probe the GPU device\n");
+    pci_device_ =
+        GetPciAccessLibApi()->pci_device_find_by_slot(pci_domain, pci_location_id >> 8, pci_location_id & 0xFF, 0);
+    if (!pci_device_ || GetPciAccessLibApi()->pci_device_probe(pci_device_)) fatal("failed to probe the GPU device\n");

-  // Look for a region between 256KB and 4096KB, 32-bit, non IO, and non prefetchable.
-  for (size_t region = 0; region < sizeof(pci_device::regions) / sizeof(pci_device::regions[0]);
-       ++region)
-    if (pci_device_->regions[region].is_64 == 0 &&
-        pci_device_->regions[region].is_prefetchable == 0 &&
-        pci_device_->regions[region].is_IO == 0 &&
-        pci_device_->regions[region].size >= (256UL * 1024) &&
-        pci_device_->regions[region].size <= (4096UL * 1024)) {
-      pci_memory_size_ = pci_device_->regions[region].size;
-      if (pci_device_map_range(pci_device_, pci_device_->regions[region].base_addr,
-                               pci_device_->regions[region].size, PCI_DEV_MAP_FLAG_WRITABLE,
-                               (void**)&pci_memory_))
-        fatal("failed to map the registers\n");
+    // Look for a region between 256KB and 4096KB, 32-bit, non IO, and non prefetchable.
+    for (size_t region = 0; region < sizeof(pci_device::regions) / sizeof(pci_device::regions[0]);
+        ++region) {
+      if (pci_device_->regions[region].is_64 == 0 &&
+          pci_device_->regions[region].is_prefetchable == 0 &&
+          pci_device_->regions[region].is_IO == 0 &&
+          pci_device_->regions[region].size >= (256UL * 1024) &&
+          pci_device_->regions[region].size <= (4096UL * 1024)) {
+        pci_memory_size_ = pci_device_->regions[region].size;
+        if (GetPciAccessLibApi()->pci_device_map_range(pci_device_, pci_device_->regions[region].base_addr,
+                                pci_device_->regions[region].size, PCI_DEV_MAP_FLAG_WRITABLE,
+                                (void**)&pci_memory_))
+          fatal("failed to map the registers\n");
+      }
    }

-  if (pci_memory_ == nullptr) fatal("could not find the pci memory address\n");
+    if (pci_memory_ == nullptr) fatal("could not find the pci memory address\n");
+  }

 device_specific_init:
  // FIXME: detect the gfxip and call the correct routine.
@@ -187,7 +192,7 @@ device_specific_init:
 }

 device_t::~device_t() {
-  if (pci_memory_ && pci_device_unmap_range(pci_device_, pci_memory_, pci_memory_size_)) {
+  if (pci_memory_ && GetPciAccessLibApi()->pci_device_unmap_range(pci_device_, pci_memory_, pci_memory_size_)) {
    warning("failed to unmap the pci memory\n");
  }
 }
@@ -0,0 +1,134 @@
+/* Copyright (c) 2022 Advanced Micro Devices, Inc.
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE. */
+
+#include "libpci_helper.h"
+
+#include <cstdio>
+#include <dlfcn.h>
+#include "exception.h"
+
+
+namespace rocprofiler {
+PcieAccessApi* api;
+void* libpciaccess_handle;
+
+// A function just to load all the symbols you need
+PcieAccessApi* LoadPcieAccessLibAPI() {
+  char* libpciaccess_tool = getenv("LIBPCI_TOOL_LIB");
+  if (libpciaccess_tool == nullptr) {
+    libpciaccess_tool = "libpciaccess.so";
+  }
+  libpciaccess_handle = dlopen(libpciaccess_tool, RTLD_NOW);
+  if (libpciaccess_handle == nullptr) {
+    fprintf(stderr, "ROCProfiler: can't load libpciaccess library \"%s\"\n", libpciaccess_tool);
+    fprintf(stderr, "%s\n", dlerror());
+    abort();
+  }
+  dlerror(); /* Clear any existing error */
+
+  api = new PcieAccessApi();
+  api->pci_device_find_by_slot =
+      reinterpret_cast<pci_device_find_by_slot_handler_t>(dlsym(
+          libpciaccess_handle, "pci_device_find_by_slot"));
+  if (api->pci_device_find_by_slot == nullptr) {
+    fprintf(stderr,
+        "ROCProfiler: libpciaccess library corrupted, pci_device_find_by_slot() method is "
+        "expected\n");
+    fprintf(stderr, "%s\n", dlerror());
+    abort();
+  }
+  api->pci_device_probe =
+      reinterpret_cast<pci_device_probe_handler_t>(dlsym(
+          libpciaccess_handle, "pci_device_probe"));
+  if (api->pci_device_probe == nullptr) {
+    fprintf(stderr,
+        "ROCProfiler: libpciaccess library corrupted, pci_device_probe() method is "
+        "expected\n");
+    fprintf(stderr, "%s\n", dlerror());
+    abort();
+  }
+  api->pci_device_map_range =
+      reinterpret_cast<pci_device_map_range_handler_t>(dlsym(
+          libpciaccess_handle, "pci_device_map_range"));
+  if (api->pci_device_map_range == nullptr) {
+    fprintf(stderr,
+        "ROCProfiler: libpciaccess library corrupted, pci_device_map_range() method is "
+        "expected\n");
+    fprintf(stderr, "%s\n", dlerror());
+    abort();
+  }
+  api->pci_device_unmap_range =
+      reinterpret_cast<pci_device_unmap_range_handler_t>(dlsym(
+          libpciaccess_handle, "pci_device_unmap_range"));
+
+  if (api->pci_device_unmap_range == nullptr) {
+    fprintf(stderr,
+        "ROCProfiler: libpciaccess library corrupted, pci_device_unmap_range() method is "
+        "expected\n");
+    fprintf(stderr, "%s\n", dlerror());
+    abort();
+  }
+  api->pci_system_init =
+      reinterpret_cast<pci_system_init_handler_t>(dlsym(
+          libpciaccess_handle, "pci_system_init"));
+  if (api->pci_system_init == nullptr) {
+    fprintf(stderr,
+        "ROCProfiler: libpciaccess library corrupted, pci_system_init() method is "
+        "expected\n");
+    fprintf(stderr, "%s\n", dlerror());
+    abort();
+  }
+  api->pci_system_cleanup =
+      reinterpret_cast<pci_system_cleanup_handler_t>(dlsym(
+          libpciaccess_handle, "pci_system_cleanup"));
+  if (api->pci_system_cleanup == nullptr) {
+    fprintf(stderr,
+        "ROCProfiler: libpciaccess library corrupted, pci_system_cleanup() method is "
+        "expected\n");
+    fprintf(stderr, "%s\n", dlerror());
+    abort();
+  }
+  return api;
+}
+
+const PcieAccessApi* GetPciAccessLibApi() {
+  if (api == nullptr) {
+    api = LoadPcieAccessLibAPI();
+  }
+  return api;
+}
+
+void UnLoadPcieAccessLibAPI() {
+  if (api == nullptr) {
+    return;
+  }
+  api->pci_device_find_by_slot = nullptr;
+  api->pci_device_probe = nullptr;
+  api->pci_device_map_range = nullptr;
+  api->pci_device_unmap_range = nullptr;
+  api->pci_system_init = nullptr;
+  api->pci_system_cleanup = nullptr;
+  delete api;
+  api = nullptr;
+  if (dlclose(libpciaccess_handle) != 0) {
+    warning("Warning: Failed to close libpciaccess handle: %s\n", dlerror());
+  }
+}
+}  // namespace rocprofiler
@@ -0,0 +1,51 @@
+/* Copyright (c) 2022 Advanced Micro Devices, Inc.
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE. */
+#pragma once
+
+#include <pciaccess.h>
+
+namespace rocprofiler {
+    typedef struct pci_device * (*pci_device_find_by_slot_handler_t)(uint32_t, 
+        uint32_t, uint32_t, uint32_t);
+    typedef int (*pci_device_probe_handler_t)(struct pci_device *);
+    typedef int (*pci_device_map_range_handler_t)(struct pci_device *, pciaddr_t,
+        pciaddr_t, unsigned, void **);
+    typedef int (*pci_device_unmap_range_handler_t) (struct pci_device *, void *,
+        pciaddr_t);
+    typedef int (*pci_system_init_handler_t)();
+    typedef void (*pci_system_cleanup_handler_t)();
+
+    struct PcieAccessApi{
+        pci_device_find_by_slot_handler_t pci_device_find_by_slot;
+        pci_device_probe_handler_t pci_device_probe;
+        pci_device_map_range_handler_t pci_device_map_range;
+        pci_device_unmap_range_handler_t pci_device_unmap_range;
+        pci_system_init_handler_t pci_system_init;
+        pci_system_cleanup_handler_t pci_system_cleanup;
+    };
+
+    
+    // A function just to load all the libpciaccess symbols
+    PcieAccessApi* LoadPcieAccessLibAPI();
+    // A simple getter function for easy access to the API table
+    const PcieAccessApi* GetPciAccessLibApi();
+    // Sets all API table functions to null and calls dlclose
+    void UnLoadPcieAccessLibAPI();
+}  // namespace rocprofiler