Merge amd-dev into amd-master 20240719

Signed-off-by: Maisam Arif <Maisam.Arif@amd.com>
Change-Id: If86d0b42820c0faf1a6ea4525aec6d11bf57a510


[ROCm/amdsmi commit: c9a113eecf]

projects/amdsmi/CHANGELOG.md

@@ -145,6 +145,9 @@ ASIC products. This requires users to update any ABIs using this structure.
 
 ### Fixes
 
+- **Fixed Leftover Mutex deadlock when running multiple instances of the CLI tool**.  
+When running `amd-smi reset --gpureset --gpu all` and then running an instance of `amd-smi static` (or any other subcommand that access the GPUs) a mutex would lock and not return requiring either a clear of the mutex in /dev/shm or rebooting the machine.
+
 - **Fixed multiple processes not being registered in `amd-smi process` with json and csv format**.  
 Multiple process outputs in the CLI tool were not being registered correctly. The json output did not handle multiple processes and is now in a new valid json format:
 

projects/amdsmi/amdsmi_cli/amdsmi_cli.py

@@ -42,7 +42,7 @@ def _print_error(e, destination):
     if destination in ['stdout', 'json', 'csv']:
         print(e)
     else:
-        f = open(destination, "w")
+        f = open(destination, "w", encoding="utf-8")
         f.write(e)
         f.close()
         print("Error occured. Result written to " + str(destination) + " file")

projects/amdsmi/amdsmi_cli/amdsmi_commands.py

@@ -122,7 +122,7 @@ class AMDSMICommands():
             if self.logger.destination == 'stdout':
                 print(human_readable_output)
             else:
-                with self.logger.destination.open('a') as output_file:
+                with self.logger.destination.open('a', encoding="utf-8") as output_file:
                     output_file.write(human_readable_output + '\n')
         elif self.logger.is_json_format() or self.logger.is_csv_format():
             self.logger.print_output()
@@ -4300,11 +4300,19 @@ class AMDSMICommands():
             self.logger.table_header += 'DOUBLE_ECC'.rjust(12)
 
             try:
-                pcie_replay = amdsmi_interface.amdsmi_get_gpu_pci_replay_counter(args.gpu)
-                monitor_values['pcie_replay'] = pcie_replay
+                pcie_metric = amdsmi_interface.amdsmi_get_pcie_info(args.gpu)['pcie_metric']
+                logging.debug("PCIE Metric for %s | %s", gpu_id, pcie_metric)
+                monitor_values['pcie_replay'] = pcie_metric['pcie_replay_count']
             except amdsmi_exception.AmdSmiLibraryException as e:
                 monitor_values['pcie_replay'] = "N/A"
-                logging.debug("Failed to get pcie replay counter on gpu %s | %s", gpu_id, e.get_error_info())
+                logging.debug("Failed to get gpu_metrics pcie replay counter on gpu %s | %s", gpu_id, e.get_error_info())
+
+            if monitor_values['pcie_replay'] == "N/A":
+                try:
+                    pcie_replay = amdsmi_interface.amdsmi_get_gpu_pci_replay_counter(args.gpu)
+                    monitor_values['pcie_replay'] = pcie_replay
+                except amdsmi_exception.AmdSmiLibraryException as e:
+                    logging.debug("Failed to get sysfs pcie replay counter on gpu %s | %s", gpu_id, e.get_error_info())
 
             self.logger.table_header += 'PCIE_REPLAY'.rjust(13)
         if args.vram_usage:

projects/amdsmi/amdsmi_cli/amdsmi_helpers.py

@@ -22,6 +22,7 @@
 
 import logging
 import math
+import os
 import platform
 import sys
 import time
@@ -58,19 +59,23 @@ class AMDSMIHelpers():
             self._is_linux = True
             logging.debug(f"AMDSMIHelpers: Platform is linux:{self._is_linux}")
 
-            output = run(["lscpu"], stdout=PIPE, stderr=STDOUT, encoding="UTF-8").stdout
-            if "hypervisor" not in output:
-                self._is_baremetal = True
-            else:
-                self._is_virtual_os = True
+            try:
+                with open('/proc/cpuinfo', 'r') as f:
+                    if 'hypervisor' in f.read():
+                        self._is_virtual_os = True
+            except IOError:
+                pass
+
+            self._is_baremetal = not self._is_virtual_os
 
             # Check for passthrough system filtering by device id
-            output = run(["lspci", "-nn"], stdout=PIPE, stderr=STDOUT, encoding="UTF-8").stdout
+            output = self.get_pci_device_ids()
             passthrough_device_ids = ["7460", "73c8", "74a0", "74a1", "74a2"]
-            if any(device_id in output for device_id in passthrough_device_ids):
-                self._is_baremetal = True
-                self._is_virtual_os = False
-                self._is_passthrough = True
+            if any(('0x' + device_id) in output for device_id in passthrough_device_ids):
+                if self._is_virtual_os:
+                    self._is_baremetal = True
+                    self._is_virtual_os = False
+                    self._is_passthrough = True
 
 
     def os_info(self, string_format=True):
@@ -783,3 +788,17 @@ class AMDSMIHelpers():
             int : converted SI unit of value requested
         """
         return int(float(val) * unit_in / unit_out)
+
+    def get_pci_device_ids(self) -> set[str]:
+        pci_devices_path = "/sys/bus/pci/devices"
+        pci_devices: set[str] = set()
+        for device in os.listdir(pci_devices_path):
+            subsystem_device_path = os.path.join(pci_devices_path, device, "subsystem_device")
+            try:
+                with open(subsystem_device_path, 'r') as f:
+                    subsystem_device = f.read().strip()
+                    pci_devices.add(subsystem_device)
+            except Exception as _:
+                continue
+        return pci_devices
+

projects/amdsmi/amdsmi_cli/amdsmi_logger.py

@@ -480,10 +480,10 @@ class AMDSMILogger():
                 print(json_std_output)
         else: # Write output to file
             if watching_output: # Flush the full JSON output to the file on watch command completion
-                with self.destination.open('w') as output_file:
+                with self.destination.open('w', encoding="utf-8") as output_file:
                     json.dump(self.watch_output, output_file, indent=4)
             else:
-                with self.destination.open('a') as output_file:
+                with self.destination.open('a', encoding="utf-8") as output_file:
                     json.dump(json_output, output_file, indent=4)
 
 
@@ -516,7 +516,7 @@ class AMDSMILogger():
                 print(str(csv_stdout_output))
         else:
             if watching_output:
-                with self.destination.open('w', newline = '') as output_file:
+                with self.destination.open('w', newline = '', encoding="utf-8") as output_file:
                     if self.watch_output:
                         csv_keys = set()
                         for output in self.watch_output:
@@ -534,7 +534,7 @@ class AMDSMILogger():
                         writer.writeheader()
                         writer.writerows(self.watch_output)
             else:
-                with self.destination.open('a', newline = '') as output_file:
+                with self.destination.open('a', newline = '', encoding="utf-8") as output_file:
                     # Get the header as a list of the first element to maintain order
                     csv_header = stored_csv_output[0].keys()
                     writer = csv.DictWriter(output_file, csv_header)
@@ -622,7 +622,7 @@ class AMDSMILogger():
                     print()
         else:
             if watching_output:
-                with self.destination.open('w', newline = '') as output_file:
+                with self.destination.open('w', newline = '', encoding="utf-8") as output_file:
                     primary_csv_output = []
                     secondary_csv_output = []
                     if self.watch_output:
@@ -687,7 +687,7 @@ class AMDSMILogger():
                             writer.writeheader()
                             writer.writerows(secondary_csv_output)
             else:
-                with self.destination.open('a', newline = '') as output_file:
+                with self.destination.open('a', newline = '', encoding="utf-8") as output_file:
                     if primary_csv_output:
                         # Get the header as a list of the first element to maintain order
                         csv_header = primary_csv_output[0].keys()
@@ -724,13 +724,13 @@ class AMDSMILogger():
                 print(human_readable_output.encode('ascii', 'ignore').decode('ascii'))
         else:
             if watching_output:
-                with self.destination.open('w') as output_file:
+                with self.destination.open('w', encoding="utf-8") as output_file:
                     human_readable_output = ''
                     for output in self.watch_output:
                         human_readable_output += self._convert_json_to_human_readable(output)
                     output_file.write(human_readable_output + '\n')
             else:
-                with self.destination.open('a') as output_file:
+                with self.destination.open('a', encoding="utf-8") as output_file:
                     output_file.write(human_readable_output + '\n')
 
 
@@ -806,7 +806,7 @@ class AMDSMILogger():
                     print("\n")
         else:
             if watching_output: # Write all stored watched output to a file
-                with self.destination.open('w') as output_file:
+                with self.destination.open('w', encoding="utf-8") as output_file:
                     primary_table = ''
                     secondary_table = ''
                     # Add process_list to the secondary_table
@@ -851,6 +851,6 @@ class AMDSMILogger():
                     if secondary_table:
                         output_file.write("\n" + secondary_table)
             else: # Write all singular output to a file
-                with self.destination.open('a') as output_file:
+                with self.destination.open('a', encoding="utf-8") as output_file:
                     output_file.write(primary_table + '\n')
                     output_file.write(secondary_table)

projects/amdsmi/amdsmi_cli/amdsmi_parser.py

@@ -140,7 +140,10 @@ class AMDSMIParser(argparse.ArgumentParser):
             return int(int_value)
 
         outputformat = self.helpers.get_output_format()
-        raise amdsmi_cli_exceptions.AmdSmiInvalidParameterValueException(int_value, outputformat)
+        if int_value == "":
+            raise amdsmi_cli_exceptions.AmdSmiMissingParameterValueException(int_value, outputformat)
+        else:
+            raise amdsmi_cli_exceptions.AmdSmiInvalidParameterValueException(int_value, outputformat)
 
 
     def _positive_int(self, int_value):
@@ -150,7 +153,10 @@ class AMDSMIParser(argparse.ArgumentParser):
                 return int(int_value)
 
         outputformat = self.helpers.get_output_format()
-        raise amdsmi_cli_exceptions.AmdSmiInvalidParameterValueException(int_value, outputformat)
+        if int_value == "":
+            raise amdsmi_cli_exceptions.AmdSmiMissingParameterValueException(int_value, outputformat)
+        else:
+            raise amdsmi_cli_exceptions.AmdSmiInvalidParameterValueException(int_value, outputformat)
 
 
     def _is_valid_string(self, string_value):
@@ -160,8 +166,10 @@ class AMDSMIParser(argparse.ArgumentParser):
             return string_value
 
         outputformat = self.helpers.get_output_format()
-        raise amdsmi_cli_exceptions.AmdSmiInvalidParameterValueException(string_value, outputformat)
-
+        if string_value == "":
+            raise amdsmi_cli_exceptions.AmdSmiMissingParameterValueException(string_value, outputformat)
+        else:
+            raise amdsmi_cli_exceptions.AmdSmiInvalidParameterValueException(string_value, outputformat)
 
     def _check_output_file_path(self):
         """ Argument action validator:

projects/amdsmi/py-interface/amdsmi_interface.py

@@ -442,13 +442,14 @@ class AmdSmiEventReader:
         for i in range(0, num_elem):
             unique_event_values = set(event.value for event in AmdSmiEvtNotificationType)
             if self.event_info[i].event in unique_event_values:
-                ret.append(
-                    {
-                        "processor_handle": self.event_info[i].processor_handle,
-                        "event": AmdSmiEvtNotificationType(self.event_info[i].event).name,
-                        "message": self.event_info[i].message.decode("utf-8"),
-                    }
-                )
+                if AmdSmiEvtNotificationType(self.event_info[i].event).name != "NONE":
+                    ret.append(
+                        {
+                            "processor_handle": self.event_info[i].processor_handle,
+                            "event": AmdSmiEvtNotificationType(self.event_info[i].event).name,
+                            "message": self.event_info[i].message.decode("utf-8"),
+                        }
+                    )
 
         return ret
 

projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi.h

@@ -149,6 +149,7 @@ typedef enum {
                                          //!< information can be retrieved. By
                                          //!< default, only AMD devices are
                                          //!<  enumerated by RSMI.
+  RSMI_INIT_FLAG_THRAD_ONLY_MUTEX = 0x400000000000000,   //!< The mutex limit to thread
   RSMI_INIT_FLAG_RESRV_TEST1 = 0x800000000000000,  //!< Reserved for test
 } rsmi_init_flags_t;
 

projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi_main.h

@@ -88,6 +88,9 @@ class RocmSMI {
 
     void set_init_options(uint64_t options) {init_options_ = options;}
     uint64_t init_options() const {return init_options_;}
+    uint64_t is_thread_only_mutex() const {
+      return init_options_ & RSMI_INIT_FLAG_THRAD_ONLY_MUTEX;
+    }
 
     uint32_t euid() const {return euid_;}
 

projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi_utils.h

@@ -242,7 +242,7 @@ class ScopeGuard {
   __forceinline ~ScopeGuard() {
     if (!dismiss_) release_();
   }
-  __forceinline ScopeGuard& operator=(const ScopeGuard& rhs) {
+  __forceinline ScopeGuard& operator=(ScopeGuard& rhs) {
     dismiss_ = rhs.dismiss_;
     release_ = rhs.release_;
     rhs.dismiss_ = true;

projects/amdsmi/rocm_smi/src/rocm_smi.cc

@@ -3435,7 +3435,8 @@ rsmi_dev_gpu_reset(uint32_t dv_ind) {
   ss << __PRETTY_FUNCTION__ << "| ======= start =======";
   LOG_TRACE(ss);
   REQUIRE_ROOT_ACCESS
-  DEVICE_MUTEX
+  // No longer using DEVICE_MUTEX as it blocks long running processes
+  // DEVICE_MUTEX
 
   rsmi_status_t ret;
   uint64_t status_code = 0;

projects/amdsmi/third_party/shared_mutex/shared_mutex.cc

@@ -41,6 +41,22 @@ THE SOFTWARE.
 #include <vector>
 
 #include "rocm_smi/rocm_smi_exception.h"
+#include "rocm_smi/rocm_smi_main.h"
+
+#define THREAD_ONLY_ENV_VAR "RSMI_MUTEX_THREAD_ONLY"
+#define MUTEX_TIME_OUT_ENV_VAR "RSMI_MUTEX_TIMEOUT"
+#define DEFAULT_MUTEX_TIMEOUT_SECONDS 5
+
+static int GetEnvVarUInteger(const char *ev_str) {
+  ev_str = getenv(ev_str);
+
+  if (ev_str) {
+    const int ret = atoi(ev_str);
+    return ret;
+  }
+
+  return -1;
+}
 
 // find which processes are using the file by searching /proc/*/fd
 static std::vector<std::string> lsof(const char* filename) {
@@ -48,10 +64,13 @@ static std::vector<std::string> lsof(const char* filename) {
   DIR *dp = nullptr;
   std::vector<std::string> process_id;
 
+  pid_t cur_pid = getpid();
   dp = opendir("/proc");
   if (dp != nullptr) {
     while ((entry = readdir(dp))) {
       std::string id(entry->d_name);
+      // ignore current process
+      if (id == std::to_string(cur_pid)) continue;
       // the process id should be a number
       if (std::all_of(id.begin(), id.end(), ::isdigit)) {
         process_id.push_back(entry->d_name);
@@ -78,9 +97,52 @@ static std::vector<std::string> lsof(const char* filename) {
   return matched_process;
 }
 
-shared_mutex_t shared_mutex_init(const char *name, mode_t mode, bool retried) {
-  shared_mutex_t mutex = {NULL, 0, NULL, 0};
+// RSMI_MUTEX_THREAD_ONLY = 1 to enable thread safe mutex
+shared_mutex_t init_thread_safe_only(const char *name) {
+  shared_mutex_t mutex = {nullptr, 0, nullptr, 0};
   errno = 0;
+  mutex.shm_fd = -1;
+  mutex.created = 0;
+  pthread_mutex_t *mutex_ptr =  new pthread_mutex_t();
+
+  pthread_mutexattr_t attr;
+  if (pthread_mutexattr_init(&attr)) {
+    perror("pthread_mutexattr_init");
+    return mutex;
+  }
+  if (pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED)) {
+    perror("pthread_mutexattr_setpshared");
+    return mutex;
+  }
+
+  if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE)) {
+    perror("pthread_mutexattr_settype");
+    return mutex;
+  }
+  if (pthread_mutexattr_setrobust(&attr, PTHREAD_MUTEX_ROBUST)) {
+    perror("pthread_mutexattr_setrobust");
+    return mutex;
+  }
+  if (pthread_mutex_init(mutex_ptr, &attr)) {
+    perror("pthread_mutex_init");
+    return mutex;
+  }
+
+  mutex.ptr = mutex_ptr;
+  mutex.name = reinterpret_cast<char *>(malloc(NAME_MAX+1));
+  (void)snprintf(mutex.name, NAME_MAX + 1, "%s", name);
+  return mutex;
+}
+
+shared_mutex_t shared_mutex_init(const char *name, mode_t mode, bool retried) {
+  shared_mutex_t mutex = {nullptr, 0, nullptr, 0};
+  errno = 0;
+
+  amd::smi::RocmSMI& smi = amd::smi::RocmSMI::getInstance();
+
+  if (GetEnvVarUInteger(THREAD_ONLY_ENV_VAR) == 1 || smi.is_thread_only_mutex()) {
+    return init_thread_safe_only(name);
+  }
 
   // Open existing shared memory object, or create one.
   // Two separate calls are needed here, to mark fact of creation
@@ -109,7 +171,7 @@ shared_mutex_t shared_mutex_init(const char *name, mode_t mode, bool retried) {
 
   // Map pthread mutex into the shared memory.
   void *addr = mmap(
-    NULL,
+    nullptr,
     sizeof(pthread_mutex_t),
     PROT_READ|PROT_WRITE,
     MAP_SHARED,
@@ -127,61 +189,15 @@ shared_mutex_t shared_mutex_init(const char *name, mode_t mode, bool retried) {
   // acquire it in 5 sec., re-do everything.
   struct timespec expireTime;
   clock_gettime(CLOCK_REALTIME, &expireTime);
-  expireTime.tv_sec += 5;
+  int time_out = GetEnvVarUInteger(MUTEX_TIME_OUT_ENV_VAR);
+  time_out = time_out < DEFAULT_MUTEX_TIMEOUT_SECONDS ? DEFAULT_MUTEX_TIMEOUT_SECONDS: time_out;
+  expireTime.tv_sec += time_out;
 
+  pid_t cur_pid = getpid();
   int ret;
 
-  ret = pthread_mutex_timedlock(mutex_ptr, &expireTime);
-
-  if (ret == EOWNERDEAD) {
-    ret = pthread_mutex_consistent(mutex_ptr);
-    // This function should not fail unless mutex_ptr is not robust
-    // or mutex_ptr is not in an inconsistent state. Neither scenario
-    // should ever be true at this point in the code.
-    assert(!ret);
-
-    // ...but if there are undocumented failure cases for
-    // pthread_mutex_consistent() handle them for release builds.
-    if (ret) {
-      fprintf(stderr, "pthread_mutex_consistent() returned %d\n", ret);
-      free(mutex.name);
-
-      throw amd::smi::rsmi_exception(RSMI_STATUS_BUSY, __FUNCTION__);
-      return mutex;
-    }
-  } else if (ret || (mutex.created == 0 &&
-                     reinterpret_cast<shared_mutex_t *>(addr)->ptr == NULL)) {
-    // Something is out of sync.
-
-    // When process crash before unlock the mutex, the mutex is in bad status.
-    // reset the mutex if no process is using it, and then retry lock
-    if (!retried) {
-      std::vector<std::string> ids = lsof(name);
-      if (ids.size() == 0) {  // no process is using it
-        memset(mutex_ptr, 0, sizeof(pthread_mutex_t));
-        // Set mutex.created == 1 so that it can be initialized latter.
-        mutex.created = 1;
-        free(mutex.name);
-        return shared_mutex_init(name, mode, true);
-      }
-    }
-
-    fprintf(stderr, "pthread_mutex_timedlock() returned %d\n", ret);
-    perror("Failed to initialize RSMI device mutex after 5 seconds. Previous "
-     "execution may not have shutdown cleanly. To fix problem, stop all "
-     "rocm_smi programs, and then delete the rocm_smi* shared memory files in"
-                                                                " /dev/shm.");
-    free(mutex.name);
-
-    throw amd::smi::rsmi_exception(RSMI_STATUS_BUSY, __FUNCTION__);
-    return mutex;
-  } else {
-    if (pthread_mutex_unlock(mutex_ptr)) {
-      perror("pthread_mutex_unlock");
-    }
-  }
-
-  if (mutex.created) {
+  // also need to set the attribute when retried as the mutex is re-initialized.
+  if (mutex.created || retried) {
     pthread_mutexattr_t attr;
     if (pthread_mutexattr_init(&attr)) {
       perror("pthread_mutexattr_init");
@@ -206,6 +222,71 @@ shared_mutex_t shared_mutex_init(const char *name, mode_t mode, bool retried) {
     }
   }
 
+  ret = pthread_mutex_timedlock(mutex_ptr, &expireTime);
+
+
+  if (ret == EOWNERDEAD) {
+    ret = pthread_mutex_consistent(mutex_ptr);
+    // This function should not fail unless mutex_ptr is not robust
+    // or mutex_ptr is not in an inconsistent state. Neither scenario
+    // should ever be true at this point in the code.
+    assert(!ret);
+
+    // ...but if there are undocumented failure cases for
+    // pthread_mutex_consistent() handle them for release builds.
+    if (ret) {
+      fprintf(stderr, "pthread_mutex_consistent() returned %d\n", ret);
+      free(mutex.name);
+
+      throw amd::smi::rsmi_exception(RSMI_STATUS_BUSY, __FUNCTION__);
+      return mutex;
+    }
+
+    fprintf(stderr, "%d detected dead process, and making mutex %s consistent.\n",
+              cur_pid, name);
+    // The mutex is locked even if EOWNERDEAD was returned,and need to unlock it.
+    if (pthread_mutex_unlock(mutex_ptr)) {
+      perror("pthread_mutex_unlock");
+    }
+  } else if (ret || (mutex.created == 0 &&
+                     reinterpret_cast<shared_mutex_t *>(addr)->ptr == nullptr)) {
+    // Something is out of sync.
+
+    // When process crash before unlock the mutex, the mutex is in bad status.
+    // reset the mutex if no process is using it, and then retry lock
+    if (!retried) {
+      std::string shared_mutex_filename = "/dev/shm" + std::string(name);
+      std::vector<std::string> ids = lsof(shared_mutex_filename.c_str());
+      if (ids.size() == 0) {  // no process is using it
+        fprintf(stderr, "%d re-init the mutex %s since no one use it. ret:%d ptr:%p\n",
+              cur_pid, shared_mutex_filename, ret, reinterpret_cast<shared_mutex_t *>(addr)->ptr);
+        memset(mutex_ptr, 0, sizeof(pthread_mutex_t));
+        // Set mutex.created == 1 so that it can be initialized latter.
+        mutex.created = 1;
+        free(mutex.name);
+        return shared_mutex_init(name, mode, true);
+      }
+    }
+
+    fprintf(stderr, "pthread_mutex_timedlock() returned %d\n", ret);
+    perror("Failed to initialize RSMI device mutex after 5 seconds. Previous "
+     "execution may not have shutdown cleanly. To fix problem, stop all "
+     "rocm_smi programs, and then delete the rocm_smi* shared memory files in"
+                                                                " /dev/shm.");
+    free(mutex.name);
+
+    throw amd::smi::rsmi_exception(RSMI_STATUS_BUSY, __FUNCTION__);
+    return mutex;
+  } else {
+    const int ret = pthread_mutex_unlock(mutex_ptr);
+    if (ret) {
+      perror("pthread_mutex_unlock");
+      fprintf(stderr, "%d init_mutex %s: unlock timed lock, ret: %d\n", cur_pid, name, ret);
+    }
+  }
+
+
+
   mutex.ptr = mutex_ptr;
   mutex.name = reinterpret_cast<char *>(malloc(NAME_MAX+1));
   (void)snprintf(mutex.name, NAME_MAX + 1, "%s", name);
@@ -213,12 +294,17 @@ shared_mutex_t shared_mutex_init(const char *name, mode_t mode, bool retried) {
 }
 
 int shared_mutex_close(shared_mutex_t mutex) {
-  if (munmap(reinterpret_cast<void *>(mutex.ptr), sizeof(pthread_mutex_t))) {
+  amd::smi::RocmSMI& smi = amd::smi::RocmSMI::getInstance();
+  const bool is_thread_only = GetEnvVarUInteger(THREAD_ONLY_ENV_VAR) == 1 ||
+          smi.is_thread_only_mutex();
+  if (is_thread_only) {
+    delete mutex.ptr;
+  } else if (munmap(reinterpret_cast<void *>(mutex.ptr), sizeof(pthread_mutex_t))) {
     perror("munmap");
     return -1;
   }
-  mutex.ptr = NULL;
-  if (close(mutex.shm_fd)) {
+  mutex.ptr = nullptr;
+  if (!is_thread_only && close(mutex.shm_fd)) {
     perror("close");
     return -1;
   }
@@ -229,21 +315,26 @@ int shared_mutex_close(shared_mutex_t mutex) {
 }
 
 int shared_mutex_destroy(shared_mutex_t mutex) {
+  amd::smi::RocmSMI& smi = amd::smi::RocmSMI::getInstance();
+  const bool is_thread_only = GetEnvVarUInteger(THREAD_ONLY_ENV_VAR) == 1 ||
+          smi.is_thread_only_mutex();
   if ((errno = pthread_mutex_destroy(mutex.ptr))) {
     perror("pthread_mutex_destroy");
     return -1;
   }
-  if (munmap(reinterpret_cast<void *>(mutex.ptr), sizeof(pthread_mutex_t))) {
+  if (is_thread_only) {
+    delete mutex.ptr;
+  } else if (munmap(reinterpret_cast<void *>(mutex.ptr), sizeof(pthread_mutex_t))) {
     perror("munmap");
     return -1;
   }
-  mutex.ptr = NULL;
-  if (close(mutex.shm_fd)) {
+  mutex.ptr = nullptr;
+  if (!is_thread_only && close(mutex.shm_fd)) {
     perror("close");
     return -1;
   }
   mutex.shm_fd = 0;
-  if (shm_unlink(mutex.name)) {
+  if (!is_thread_only && shm_unlink(mutex.name)) {
     perror("shm_unlink");
     return -1;
   }