Added rsmi_dev_pci_replay_counter_get()

Also, added code to destroy/recreate mutex if we can't get a lock
within 3 seconds, when shared memory mutex is initialized.

src/rocm_smi.cc

@@ -120,8 +120,8 @@ static rsmi_status_t errno_to_rsmi_status(uint32_t err) {
   switch (err) {
     case 0:      return RSMI_STATUS_SUCCESS;
     case EACCES: return RSMI_STATUS_PERMISSION;
-    case EPERM:  return RSMI_STATUS_NOT_SUPPORTED;
-    case ENOENT:
+    case EPERM:
+    case ENOENT: return RSMI_STATUS_NOT_SUPPORTED;
     case EISDIR: return RSMI_STATUS_FILE_ERROR;
     default:     return RSMI_STATUS_UNKNOWN_ERROR;
   }
@@ -2044,3 +2044,16 @@ rsmi_version_str_get(rsmi_sw_component_t component, char *ver_str,
 
   CATCH
 }
+
+rsmi_status_t
+rsmi_dev_pci_replay_counter_get(uint32_t dv_ind, uint64_t *counter) {
+  TRY
+
+  DEVICE_MUTEX
+  rsmi_status_t ret;
+
+  ret = get_dev_value_int(amd::smi::kDevPCIEReplayCount, dv_ind, counter);
+  return ret;
+
+  CATCH
+}

src/rocm_smi_device.cc

@@ -96,6 +96,7 @@ static const char *kDevMemTotVRAMFName = "mem_info_vram_total";
 static const char *kDevMemUsedGTTFName = "mem_info_gtt_used";
 static const char *kDevMemUsedVisVRAMFName = "mem_info_vis_vram_used";
 static const char *kDevMemUsedVRAMFName = "mem_info_vram_used";
+static const char *kDevPCIEReplayCountFName = "pcie_replay_count";
 
 // Strings that are found within sysfs files
 static const char *kDevPerfLevelAutoStr = "auto";
@@ -136,6 +137,7 @@ static const std::map<DevInfoTypes, const char *> kDevAttribNameMap = {
     {kDevMemUsedGTT, kDevMemUsedGTTFName},
     {kDevMemUsedVisVRAM, kDevMemUsedVisVRAMFName},
     {kDevMemUsedVRAM, kDevMemUsedVRAMFName},
+    {kDevPCIEReplayCount, kDevPCIEReplayCountFName},
 };
 
 static const std::map<rsmi_dev_perf_level, const char *> kDevPerfLvlMap = {
@@ -202,7 +204,7 @@ int Device::openSysfsFileStream(DevInfoTypes type, T *fs, const char *str) {
 
   DBG_FILE_ERROR(sysfs_path, str);
   if (!isRegularFile(sysfs_path)) {
-    return EISDIR;
+    return ENOENT;
   }
 
   fs->open(sysfs_path);
@@ -367,6 +369,7 @@ int Device::readDevInfo(DevInfoTypes type, uint64_t *val) {
     case kDevMemUsedGTT:
     case kDevMemUsedVisVRAM:
     case kDevMemUsedVRAM:
+    case kDevPCIEReplayCount:
       ret = readDevInfoStr(type, &tempStr);
       RET_IF_NONZERO(ret);
       *val = std::stoul(tempStr, 0);

src/shared_mutex/shared_mutex.c

@@ -8,6 +8,7 @@
 #include <stdio.h> // perror
 #include <stdlib.h> // malloc, free
 #include <string.h> // strcpy
+#include <time.h>  // clock_gettime
 
 shared_mutex_t shared_mutex_init(const char *name, mode_t mode) {
   shared_mutex_t mutex = {NULL, 0, NULL, 0};
@@ -51,7 +52,17 @@ shared_mutex_t shared_mutex_init(const char *name, mode_t mode) {
     return mutex;
   }
 
-  if (mutex.created == 0 && ((shared_mutex_t *)addr)->ptr == NULL) {
+  pthread_mutex_t *mutex_ptr =  (pthread_mutex_t *)addr;
+
+  // Make sure the mutex wasn't left in a locked state. If we can't
+  // acquire it in 3 sec., re-do everything.
+  struct timespec expireTime;
+  clock_gettime(CLOCK_REALTIME, &expireTime);
+  expireTime.tv_sec += 3;
+
+  int ret = pthread_mutex_timedlock(mutex_ptr, &expireTime);
+  
+  if (ret || (mutex.created == 0 && ((shared_mutex_t *)addr)->ptr == NULL)) {
     // Something is out of sync. Unlink shm and start over.
     if (shm_unlink(name)) {
       mutex.shm_fd = 0;
@@ -60,9 +71,12 @@ shared_mutex_t shared_mutex_init(const char *name, mode_t mode) {
     free(mutex.name);
 
     return shared_mutex_init(name, mode);
+  } else {
+    if (pthread_mutex_unlock(mutex_ptr)) {
+      perror("pthread_mutex_unlock");
+    }
   }
 
-  pthread_mutex_t *mutex_ptr =  (pthread_mutex_t *)addr;
 
   if (mutex.created) {
     pthread_mutexattr_t attr;

src/shared_mutex/shared_mutex.h

@@ -1,3 +1,6 @@
+// NOLINT(legal/copyright)
+// See LICENSE file
+
 #ifndef SRC_SHARED_MUTEX_SHARED_MUTEX_H_
 #define SRC_SHARED_MUTEX_SHARED_MUTEX_H_