/*
 * =============================================================================
 * The University of Illinois/NCSA
 * Open Source License (NCSA)
 *
 * Copyright (c) 2017-2023, Advanced Micro Devices, Inc.
 * All rights reserved.
 *
 * Developed by:
 *
 *                 AMD Research and AMD ROC Software Development
 *
 *                 Advanced Micro Devices, Inc.
 *
 *                 www.amd.com
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal with the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 *  - Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimers.
 *  - Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimers in
 *    the documentation and/or other materials provided with the distribution.
 *  - Neither the names of <Name of Development Group, Name of Institution>,
 *    nor the names of its contributors may be used to endorse or promote
 *    products derived from this Software without specific prior written
 *    permission.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
 * THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS WITH THE SOFTWARE.
 *
 */

#include <fcntl.h>
#include <poll.h>
#include <pthread.h>
#include <cstddef>
#include <string>
#include <sys/ioctl.h>
#include <sys/utsname.h>
#include <unistd.h>

#include <algorithm>
#include <bitset>
#include <cassert>
#include <cctype>
#include <cerrno>
#include <cstdint>
#include <cstring>
#include <fstream>
#include <iostream>
#include <map>
#include <sstream>
#include <vector>
#include <unordered_map>
#include <unordered_set>

#include "rocm_smi/rocm_smi_common.h"  // Should go before rocm_smi.h
#include "rocm_smi/rocm_smi.h"
#include "rocm_smi/rocm_smi_main.h"
#include "rocm_smi/rocm_smi_device.h"
#include "rocm_smi/rocm_smi_utils.h"
#include "rocm_smi/rocm_smi_exception.h"
#include "rocm_smi/rocm_smi_counters.h"
#include "rocm_smi/rocm_smi_kfd.h"
#include "rocm_smi/rocm_smi_io_link.h"
#include "rocm_smi/rocm_smi64Config.h"
#include "rocm_smi/rocm_smi_logger.h"

using amd::smi::monitorTypesToString;
using amd::smi::getRSMIStatusString;
using amd::smi::AMDGpuMetricsUnitType_t;
using amd::smi::AMDGpuMetricTypeId_t;
auto &devInfoTypesStrings = amd::smi::RocmSMI::devInfoTypesStrings;

static const uint32_t kMaxOverdriveLevel = 20;
static const float kEnergyCounterResolution = 15.3F;

static const std::map<rsmi_clk_type_t, std::string> kClkStateMap = {
  { RSMI_CLK_TYPE_SYS, "SCLK" },
  { RSMI_CLK_TYPE_DF, "DFCLK" },
  { RSMI_CLK_TYPE_DCEF, "DCEFCLK" },
  { RSMI_CLK_TYPE_SOC, "SOCCLK" },
  { RSMI_CLK_TYPE_MEM, "MCLK" },
  { RSMI_CLK_TYPE_PCIE, "PCIECLK" },
};

static const std::map<rsmi_clk_type_t, amd::smi::DevInfoTypes> kClkTypeMap = {
  { RSMI_CLK_TYPE_SYS, amd::smi::kDevGPUSClk },
  { RSMI_CLK_TYPE_MEM, amd::smi::kDevGPUMClk },
  { RSMI_CLK_TYPE_DF, amd::smi::kDevFClk },
  { RSMI_CLK_TYPE_DCEF, amd::smi::kDevDCEFClk },
  { RSMI_CLK_TYPE_SOC, amd::smi::kDevSOCClk },
};

#define TRY try {
#define CATCH } catch (...) {return amd::smi::handleException();}

static uint64_t get_multiplier_from_str(char units_char) {
  uint32_t multiplier = 0;

  switch (units_char) {
    case 'G':   // GT or GHz
      multiplier = 1000000000;
      break;

    case 'M':   // MT or MHz
      multiplier = 1000000;
      break;

    case 'K':   // KT or KHz
    case 'V':   // default unit for voltage is mV
      multiplier = 1000;
      break;

    case 'T':   // Transactions
    case 'H':   // Hertz
    case 'm':   // mV (we will make mV the default unit for voltage)
      multiplier = 1;
      break;

    default:
      assert(false);  // Unexpected units for frequency
      throw amd::smi::rsmi_exception(RSMI_STATUS_UNEXPECTED_DATA, __FUNCTION__);
  }
  return multiplier;
}

/**
 * Parse a string of the form:
 *        "<int index>:  <int freq><freq. unit string> <|*>"
 */
static uint64_t freq_string_to_int(const std::vector<std::string> &freq_lines,
                                bool *is_curr, uint32_t lanes[], uint32_t i) {
  assert(i < freq_lines.size());
  if (i >= freq_lines.size()) {
    throw amd::smi::rsmi_exception(RSMI_STATUS_INPUT_OUT_OF_BOUNDS,
                                                                __FUNCTION__);
  }

  std::istringstream fs(freq_lines[i]);

  char junk_ch;
  int ind;
  float freq;
  std::string junk_str;
  std::string units_str;
  std::string star_str;

  if (fs.peek() == 'S') {
    // Deep Sleep frequency is only supported by some GPUs
    fs >> junk_ch;
  } else {
    // All other frequency indices are numbers
    fs >> ind;
  }
  fs >> junk_str;  // colon
  fs >> freq;
  fs >> units_str;
  fs >> star_str;

  if (freq < 0) {
    throw amd::smi::rsmi_exception(RSMI_STATUS_UNEXPECTED_SIZE, __FUNCTION__);
  }

  if (is_curr != nullptr) {
    if (freq_lines[i].find('*') != std::string::npos) {
      *is_curr = true;
    } else {
      *is_curr = false;
    }
  }
  long double multiplier = get_multiplier_from_str(units_str[0]);

  if (star_str[0] == 'x') {
    assert(lanes != nullptr && "Lanes are provided but null lanes pointer");
    if (lanes) {
      if (star_str.substr(1).empty()) {
        throw amd::smi::rsmi_exception(RSMI_STATUS_NO_DATA, __FUNCTION__);
      }

      lanes[i] =
                static_cast<uint32_t>(std::stoi(star_str.substr(1), nullptr));
    }
  }
  return static_cast<uint64_t>(freq*multiplier);
}

static void freq_volt_string_to_point(std::string in_line,
                                                     rsmi_od_vddc_point_t *pt) {
  std::istringstream fs_vlt(in_line);

  assert(pt != nullptr);
  THROW_IF_NULLPTR_DEREF(pt)

  uint32_t ind;
  float freq;
  float volts;
  std::string junk;
  std::string freq_units_str;
  std::string volts_units_str;

  fs_vlt >> ind;
  fs_vlt >> junk;  // colon
  fs_vlt >> freq;
  fs_vlt >> freq_units_str;
  fs_vlt >> volts;
  fs_vlt >> volts_units_str;

  if (freq < 0) {
    throw amd::smi::rsmi_exception(RSMI_STATUS_UNEXPECTED_SIZE, __FUNCTION__);
  }

  long double multiplier = get_multiplier_from_str(freq_units_str[0]);

  pt->frequency = static_cast<uint64_t>(freq*multiplier);

  multiplier = get_multiplier_from_str(volts_units_str[0]);
  pt->voltage = static_cast<uint64_t>(volts*multiplier);
}

static void od_value_pair_str_to_range(std::string in_line, rsmi_range_t *rg) {
  std::istringstream fs_rng(in_line);

  assert(rg != nullptr);
  THROW_IF_NULLPTR_DEREF(rg)

  std::string clk;
  float lo;
  float hi;
  std::string lo_units_str;
  std::string hi_units_str;

  fs_rng >> clk;  // This is clk + colon; e.g., "SCLK:"
  fs_rng >> lo;
  fs_rng >> lo_units_str;
  fs_rng >> hi;
  fs_rng >> hi_units_str;

  long double multiplier = get_multiplier_from_str(lo_units_str[0]);

  rg->lower_bound = static_cast<uint64_t>(lo*multiplier);

  multiplier = get_multiplier_from_str(hi_units_str[0]);
  rg->upper_bound = static_cast<uint64_t>(hi*multiplier);
}

/**
 * Parse a string of the form "<int index> <mode name string> <|*>"
 */
static rsmi_power_profile_preset_masks
power_prof_string_to_int(std::string pow_prof_line, bool *is_curr,
                                                          uint32_t *prof_ind) {
  std::istringstream fs(pow_prof_line);
  std::string mode;
  size_t tmp;

  THROW_IF_NULLPTR_DEREF(prof_ind)

  rsmi_power_profile_preset_masks_t ret = RSMI_PWR_PROF_PRST_INVALID;

  fs >> *prof_ind;
  fs >> mode;

  while (true) {
    tmp = mode.find_last_of("* :");
    if (tmp == std::string::npos) {
      break;
    }
    mode = mode.substr(0, tmp);
  }

  if (is_curr != nullptr) {
    if (pow_prof_line.find('*') != std::string::npos) {
      *is_curr = true;
    } else {
      *is_curr = false;
    }
  }

  const std::unordered_map<std::string, std::function<void()>> mode_map {
    {"BOOTUP_DEFAULT",   [&](){ ret = RSMI_PWR_PROF_PRST_BOOTUP_DEFAULT; }},
    {"3D_FULL_SCREEN",   [&](){ ret = RSMI_PWR_PROF_PRST_3D_FULL_SCR_MASK; }},
    {"POWER_SAVING",   [&](){ ret = RSMI_PWR_PROF_PRST_POWER_SAVING_MASK; }},
    {"VIDEO", [&](){ ret = RSMI_PWR_PROF_PRST_VIDEO_MASK; }},
    {"VR", [&](){ ret = RSMI_PWR_PROF_PRST_VR_MASK; }},
    {"COMPUTE", [&](){ ret = RSMI_PWR_PROF_PRST_COMPUTE_MASK; }},
    {"CUSTOM", [&](){ ret = RSMI_PWR_PROF_PRST_CUSTOM_MASK; }},
  };
  auto mode_iter = mode_map.find(mode);

  if (mode_iter != mode_map.end()) {
    mode_iter->second();
  }
  return ret;
}

static rsmi_status_t get_dev_value_str(amd::smi::DevInfoTypes type,
                                      uint32_t dv_ind, std::string *val_str) {
  assert(val_str != nullptr);
  if (val_str == nullptr) {
    return RSMI_STATUS_INVALID_ARGS;
  }
  GET_DEV_FROM_INDX
  int ret = dev->readDevInfo(type, val_str);

  return amd::smi::ErrnoToRsmiStatus(ret);
}
static rsmi_status_t get_dev_value_int(amd::smi::DevInfoTypes type,
                                         uint32_t dv_ind, uint64_t *val_int) {
  assert(val_int != nullptr);
  if (val_int == nullptr) {
    return RSMI_STATUS_INVALID_ARGS;
  }
  GET_DEV_FROM_INDX
  int ret = dev->readDevInfo(type, val_int);

  return amd::smi::ErrnoToRsmiStatus(ret);
}

static rsmi_status_t get_dev_value_line(amd::smi::DevInfoTypes type,
                                      uint32_t dv_ind, std::string *val_str) {
  assert(val_str != nullptr);
  if (val_str == nullptr) {
    return RSMI_STATUS_INVALID_ARGS;
  }
  GET_DEV_FROM_INDX
  int ret = dev->readDevInfoLine(type, val_str);

  return amd::smi::ErrnoToRsmiStatus(ret);
}

static rsmi_status_t set_dev_value(amd::smi::DevInfoTypes type,
                                              uint32_t dv_ind, uint64_t val) {
  GET_DEV_FROM_INDX

  int ret = dev->writeDevInfo(type, val);
  return amd::smi::ErrnoToRsmiStatus(ret);
}

static rsmi_status_t get_dev_mon_value(amd::smi::MonitorTypes type,
                         uint32_t dv_ind, uint32_t sensor_ind, int64_t *val) {
  assert(val != nullptr);
  if (val == nullptr) {
    return RSMI_STATUS_INVALID_ARGS;
  }
  GET_DEV_FROM_INDX

  if (dev->monitor() == nullptr) {
    return RSMI_STATUS_NOT_SUPPORTED;
  }

  std::string val_str;

  int ret = dev->monitor()->readMonitor(type, sensor_ind, &val_str);
  if (ret) {
    return amd::smi::ErrnoToRsmiStatus(ret);
  }

  if (!amd::smi::IsInteger(val_str)) {
    std::cerr << "Expected integer value from monitor,"
                                " but got \"" << val_str << "\"" << std::endl;
    return RSMI_STATUS_UNEXPECTED_DATA;
  }

  *val = std::stoi(val_str);

  return RSMI_STATUS_SUCCESS;
}

static rsmi_status_t get_dev_mon_value(amd::smi::MonitorTypes type,
                        uint32_t dv_ind, uint32_t sensor_ind, uint64_t *val) {
  assert(val != nullptr);
  if (val == nullptr) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  GET_DEV_FROM_INDX
  if (dev->monitor() == nullptr) {
    return RSMI_STATUS_NOT_SUPPORTED;
  }
  std::string val_str;

  int ret = dev->monitor()->readMonitor(type, sensor_ind, &val_str);
  if (ret) {
    return amd::smi::ErrnoToRsmiStatus(ret);
  }

  if (!amd::smi::IsInteger(val_str)) {
    std::cerr << "Expected integer value from monitor,"
                                " but got \"" << val_str << "\"" << std::endl;
    return RSMI_STATUS_UNEXPECTED_DATA;
  }

  *val = std::stoul(val_str);

  return RSMI_STATUS_SUCCESS;
}

template <typename T>
static rsmi_status_t set_dev_mon_value(amd::smi::MonitorTypes type,
                                uint32_t dv_ind, uint32_t sensor_ind, T val) {
  GET_DEV_FROM_INDX

  if (dev->monitor() == nullptr) {
    return RSMI_STATUS_NOT_SUPPORTED;
  }
  int ret = dev->monitor()->writeMonitor(type, sensor_ind,
                                                         std::to_string(val));
  /// If the sysfs file doesn't exist, it is not supported.
  if (ret == ENOENT) {
    return rsmi_status_t::RSMI_STATUS_NOT_SUPPORTED;
  }
  return amd::smi::ErrnoToRsmiStatus(ret);
}

static rsmi_status_t get_power_mon_value(amd::smi::PowerMonTypes type,
                                      uint32_t dv_ind, uint64_t *val) {
  amd::smi::RocmSMI& smi = amd::smi::RocmSMI::getInstance();

  if (dv_ind >= smi.devices().size() || val == nullptr) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  int ret = smi.DiscoverAMDPowerMonitors();
  if (ret != 0) {
    return amd::smi::ErrnoToRsmiStatus(ret);
  }

  std::shared_ptr<amd::smi::Device> dev = smi.devices()[dv_ind];
  if (dev == nullptr || dev->monitor() == nullptr) {
    return RSMI_STATUS_NOT_SUPPORTED;
  }
  ret = dev->power_monitor()->readPowerValue(type, val);

  return amd::smi::ErrnoToRsmiStatus(ret);
}

static bool is_power_of_2(uint64_t n) {
      return n && !(n & (n - 1));
}

rsmi_status_t
rsmi_init(uint64_t flags) {
  TRY
  amd::smi::RocmSMI& smi = amd::smi::RocmSMI::getInstance();
  std::lock_guard<std::mutex> guard(*smi.bootstrap_mutex());

  if (smi.ref_count() == INT32_MAX) {
    return RSMI_STATUS_REFCOUNT_OVERFLOW;
  }

  (void)smi.ref_count_inc();

  // If smi.Initialize() throws, we should clean up and dec. ref_count_.
  // Otherwise, if no issues, the Dismiss() will prevent the ref_count_
  // decrement.
  MAKE_NAMED_SCOPE_GUARD(refGuard, [&]() { (void)smi.ref_count_dec(); });

  if (smi.ref_count() == 1) {
    try {
      smi.Initialize(flags);
    } catch(...) {
      smi.Cleanup();
      throw amd::smi::rsmi_exception(RSMI_STATUS_INIT_ERROR, __FUNCTION__);
    }
  }
  refGuard.Dismiss();

  return RSMI_STATUS_SUCCESS;
  CATCH
}

// A call to rsmi_shut_down is not technically necessary at this time,
// but may be in the future.
rsmi_status_t
rsmi_shut_down(void) {
  TRY

  amd::smi::RocmSMI& smi = amd::smi::RocmSMI::getInstance();
  std::lock_guard<std::mutex> guard(*smi.bootstrap_mutex());

  if (smi.ref_count() == 0) {
    return RSMI_STATUS_INIT_ERROR;
  }

  // Release any device mutexes that are being held
#if DEBUG
  int ret = 0;
#endif
  for (uint32_t i = 0; i < smi.devices().size(); ++i) {
#if DEBUG
    ret = pthread_mutex_unlock(smi.devices()[i]->mutex());
    if (ret != EPERM) {  // We expect to get EPERM if the lock has already
                         // been released
      if (ret == 0) {
        std::cout << "WARNING: Unlocked monitor_devices lock; " <<
                    "it should have already been unlocked." << std::endl;
      } else {
      std::cout << "WARNING: pthread_mutex_unlock() returned " << ret <<
                   " for device " << i << " in rsmi_shut_down()" << std::endl;
      }
    }
#else
    (void)pthread_mutex_unlock(smi.devices()[i]->mutex());
#endif
  }

  (void)smi.ref_count_dec();

  if (smi.ref_count() == 0) {
    smi.Cleanup();
  }
  return RSMI_STATUS_SUCCESS;
  CATCH
}

rsmi_status_t
rsmi_num_monitor_devices(uint32_t *num_devices) {
  TRY
  assert(num_devices != nullptr);
  if (num_devices == nullptr) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  amd::smi::RocmSMI& smi = amd::smi::RocmSMI::getInstance();

  *num_devices = static_cast<uint32_t>(smi.devices().size());
  return RSMI_STATUS_SUCCESS;
  CATCH
}

rsmi_status_t rsmi_dev_ecc_enabled_get(uint32_t dv_ind,
                                                    uint64_t *enabled_blks) {
  TRY
  rsmi_status_t ret;
  std::string feature_line;
  std::string tmp_str;
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << " | ======= start =======";
  LOG_TRACE(ss);

  CHK_SUPPORT_NAME_ONLY(enabled_blks)

  DEVICE_MUTEX

  ret = get_dev_value_line(amd::smi::kDevErrCntFeatures, dv_ind, &feature_line);
  if (ret != RSMI_STATUS_SUCCESS) {
    ss << __PRETTY_FUNCTION__ << " | ======= end ======="
       << ", returning get_dev_value_line() response = "
       << amd::smi::getRSMIStatusString(ret);
    LOG_ERROR(ss);
    return ret;
  }

  std::istringstream fs1(feature_line);

  fs1 >> tmp_str;  // ignore
  assert(tmp_str == "feature");
  fs1 >> tmp_str;  // ignore
  assert(tmp_str == "mask:");
  fs1 >> tmp_str;

  errno = 0;
  *enabled_blks = strtoul(tmp_str.c_str(), nullptr, 16);
  assert(errno == 0);

  ss << __PRETTY_FUNCTION__ << " | ======= end ======="
     << ", returning strtoul() response = "
     << amd::smi::getRSMIStatusString(amd::smi::ErrnoToRsmiStatus(errno));
  LOG_TRACE(ss);

  return amd::smi::ErrnoToRsmiStatus(errno);
  CATCH
}

static const std::map<std::string, rsmi_ras_err_state_t> kRocmSMIStateMap = {
    {"none", RSMI_RAS_ERR_STATE_NONE},
    {"disabled", RSMI_RAS_ERR_STATE_DISABLED},
    {"parity", RSMI_RAS_ERR_STATE_PARITY},
    {"single_correctable", RSMI_RAS_ERR_STATE_SING_C},
    {"multi_uncorrectable", RSMI_RAS_ERR_STATE_MULT_UC},
    {"poison", RSMI_RAS_ERR_STATE_POISON},
    {"off", RSMI_RAS_ERR_STATE_DISABLED},
    {"on", RSMI_RAS_ERR_STATE_ENABLED},
};
static_assert(RSMI_RAS_ERR_STATE_LAST == RSMI_RAS_ERR_STATE_ENABLED,
                 "rsmi_gpu_block_t and/or above name map need to be updated"
                                                     " and then this assert");

rsmi_status_t rsmi_dev_ecc_status_get(uint32_t dv_ind, rsmi_gpu_block_t block,
                                                 rsmi_ras_err_state_t *state) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  CHK_SUPPORT_NAME_ONLY(state)

  if (!is_power_of_2(block)) {
    ss << __PRETTY_FUNCTION__ << " | ======= end ======="
       << ", ret was not power of 2 "
       << "-> reporting RSMI_STATUS_INVALID_ARGS";
    LOG_ERROR(ss);
    return RSMI_STATUS_INVALID_ARGS;
  }
  rsmi_status_t ret;
  uint64_t features_mask;

  DEVICE_MUTEX

  ret = rsmi_dev_ecc_enabled_get(dv_ind, &features_mask);

  if (ret == RSMI_STATUS_FILE_ERROR) {
    ss << __PRETTY_FUNCTION__ << " | ======= end ======="
       << ", rsmi_dev_ecc_enabled_get() ret was RSMI_STATUS_FILE_ERROR "
       << "-> reporting RSMI_STATUS_NOT_SUPPORTED";
    LOG_ERROR(ss);
    return RSMI_STATUS_NOT_SUPPORTED;
  }
  if (ret != RSMI_STATUS_SUCCESS) {
    ss << __PRETTY_FUNCTION__ << " | ======= end ======="
       << ", returning rsmi_dev_ecc_enabled_get() response = "
       << amd::smi::getRSMIStatusString(ret);
    LOG_ERROR(ss);
    return ret;
  }

  *state = (features_mask & block) ?
                     RSMI_RAS_ERR_STATE_ENABLED : RSMI_RAS_ERR_STATE_DISABLED;

  ss << __PRETTY_FUNCTION__ << " | ======= end ======="
     << ", reporting RSMI_STATUS_SUCCESS";
  LOG_TRACE(ss);
  return RSMI_STATUS_SUCCESS;
  CATCH
}

rsmi_status_t
rsmi_dev_ecc_count_get(uint32_t dv_ind, rsmi_gpu_block_t block,
                                                     rsmi_error_count_t *ec) {
  std::vector<std::string> val_vec;
  rsmi_status_t ret(RSMI_STATUS_NOT_SUPPORTED);
  std::ostringstream ss;

  TRY
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  CHK_SUPPORT_VAR(ec, block)


  amd::smi::DevInfoTypes type;
  switch (block) {
    case RSMI_GPU_BLOCK_UMC:
      type = amd::smi::kDevErrCntUMC;
      break;

    case RSMI_GPU_BLOCK_SDMA:
      type = amd::smi::kDevErrCntSDMA;
      break;

    case RSMI_GPU_BLOCK_GFX:
      type = amd::smi::kDevErrCntGFX;
      break;

    case RSMI_GPU_BLOCK_MMHUB:
      type = amd::smi::kDevErrCntMMHUB;
      break;

    case RSMI_GPU_BLOCK_PCIE_BIF:
      type = amd::smi::kDevErrCntPCIEBIF;
      break;

    case RSMI_GPU_BLOCK_HDP:
      type = amd::smi::kDevErrCntHDP;
      break;

    case RSMI_GPU_BLOCK_XGMI_WAFL:
      type = amd::smi::kDevErrCntXGMIWAFL;
      break;

    default:
      ss << __PRETTY_FUNCTION__ << " | ======= end ======="
         << ", default case -> reporting "
         << amd::smi::getRSMIStatusString(RSMI_STATUS_NOT_SUPPORTED);
      LOG_ERROR(ss);
      return RSMI_STATUS_NOT_SUPPORTED;
  }

  DEVICE_MUTEX

  ret = GetDevValueVec(type, dv_ind, &val_vec);
  if (val_vec.size() < 2 ) ret = RSMI_STATUS_FILE_ERROR;

  if (ret == RSMI_STATUS_FILE_ERROR) {
    ss << __PRETTY_FUNCTION__ << " | ======= end ======="
       << ", GetDevValueVec() ret was RSMI_STATUS_FILE_ERROR "
       << "-> reporting RSMI_STATUS_NOT_SUPPORTED";
    LOG_ERROR(ss);
    return RSMI_STATUS_NOT_SUPPORTED;
  }
  if (ret != RSMI_STATUS_SUCCESS) {
    ss << __PRETTY_FUNCTION__ << " | ======= end ======="
       << ", GetDevValueVec() ret was not RSMI_STATUS_SUCCESS"
       << " -> reporting " << amd::smi::getRSMIStatusString(ret);
    LOG_ERROR(ss);
    return ret;
  }

  std::string junk;
  std::istringstream fs1(val_vec[0]);

  fs1 >> junk;
  assert(junk == "ue:");
  fs1 >> ec->uncorrectable_err;

  std::istringstream fs2(val_vec[1]);

  fs2 >> junk;
  assert(junk == "ce:");
  fs2 >> ec->correctable_err;

  ss << __PRETTY_FUNCTION__ << " | ======= end ======="
     << ", reporting " << amd::smi::getRSMIStatusString(ret);;
  LOG_TRACE(ss);
  return ret;
  CATCH
}

rsmi_status_t
rsmi_dev_pci_id_get(uint32_t dv_ind, uint64_t *bdfid) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  GET_DEV_AND_KFDNODE_FROM_INDX
  CHK_API_SUPPORT_ONLY(bdfid, RSMI_DEFAULT_VARIANT, RSMI_DEFAULT_VARIANT)
  DEVICE_MUTEX

  *bdfid = dev->bdfid();

  uint64_t domain = 0;

  kfd_node->get_property_value("domain", &domain);

  // Replace the 16 bit domain originally set like this:
  // BDFID = ((<DOMAIN> & 0xffff) << 32) | ((<BUS> & 0xff) << 8) |
  //                        ((device& 0x1f) <<3 ) | (function & 0x7)
  // with this:
  // BDFID = ((<DOMAIN> & 0xffffffff) << 32) | ((<BUS> & 0xff) << 8) |
  //                        ((device& 0x1f) <<3 ) | (function & 0x7)

  assert((domain & 0xFFFFFFFF00000000) == 0);
  (*bdfid) &= 0xFFFF;  // Clear out the old 16 bit domain
  *bdfid |= (domain & 0xFFFFFFFF) << 32;

  ss << __PRETTY_FUNCTION__ << " | ======= end ======="
     << ", reporting RSMI_STATUS_SUCCESS";
  LOG_TRACE(ss);
  return RSMI_STATUS_SUCCESS;
  CATCH
}

rsmi_status_t
rsmi_topo_numa_affinity_get(uint32_t dv_ind, int32_t *numa_node) {
  TRY
  rsmi_status_t ret;

  CHK_SUPPORT_NAME_ONLY(numa_node)

  DEVICE_MUTEX
  std::string str_val;
  ret = get_dev_value_str(amd::smi::kDevNumaNode, dv_ind, &str_val);
  *numa_node = std::stoi(str_val, nullptr);

  return ret;
  CATCH
}

static rsmi_status_t
get_id(uint32_t dv_ind, amd::smi::DevInfoTypes typ, uint16_t *id) {
  TRY
  std::string val_str;
  uint64_t val_u64;

  assert(id != nullptr);
  if (id == nullptr) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  DEVICE_MUTEX

  rsmi_status_t ret = get_dev_value_str(typ, dv_ind, &val_str);

  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  errno = 0;
  val_u64 = strtoul(val_str.c_str(), nullptr, 16);
  assert(errno == 0);
  if (errno != 0) {
    return amd::smi::ErrnoToRsmiStatus(errno);
  }
  if (val_u64 > 0xFFFF) {
    return RSMI_STATUS_UNEXPECTED_SIZE;
  }
  *id = static_cast<uint16_t>(val_u64);

  return RSMI_STATUS_SUCCESS;
  CATCH
}

rsmi_status_t
rsmi_dev_id_get(uint32_t dv_ind, uint16_t *id) {
  std::ostringstream ss;
  rsmi_status_t ret;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  CHK_SUPPORT_NAME_ONLY(id)

  ret = get_id(dv_ind, amd::smi::kDevDevID, id);
  ss << __PRETTY_FUNCTION__ << " | ======= end ======="
     << ", reporting " << amd::smi::getRSMIStatusString(ret);
  LOG_TRACE(ss);
  return ret;
}

rsmi_status_t
rsmi_dev_xgmi_physical_id_get(uint32_t dv_ind, uint16_t *id) {
  std::ostringstream ss;
  rsmi_status_t ret;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  CHK_SUPPORT_NAME_ONLY(id)

  ret = get_id(dv_ind, amd::smi::kDevXGMIPhysicalID, id);
  ss << __PRETTY_FUNCTION__ << " | ======= end ======="
     << ", reporting " << amd::smi::getRSMIStatusString(ret);
  LOG_TRACE(ss);
  return ret;
}

rsmi_status_t
rsmi_dev_revision_get(uint32_t dv_ind, uint16_t *revision) {
  std::ostringstream outss;
  rsmi_status_t ret;
  outss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(outss);
  CHK_SUPPORT_NAME_ONLY(revision)

  ret = get_id(dv_ind, amd::smi::kDevDevRevID, revision);
  outss << __PRETTY_FUNCTION__ << " | ======= end ======="
     << ", reporting " << amd::smi::getRSMIStatusString(ret);
  LOG_TRACE(outss);
  return ret;
}

rsmi_status_t
rsmi_dev_sku_get(uint32_t dv_ind, uint16_t *id) {
  TRY
  std::ostringstream ss;
  rsmi_status_t ret;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  CHK_SUPPORT_NAME_ONLY(id)
  ret = get_id(dv_ind, amd::smi::kDevDevProdNum, id);
  ss << __PRETTY_FUNCTION__ << " | ======= end ======="
     << ", reporting " << amd::smi::getRSMIStatusString(ret);
  LOG_TRACE(ss);
  return ret;
  CATCH
}

rsmi_status_t
rsmi_dev_subsystem_id_get(uint32_t dv_ind, uint16_t *id) {
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  CHK_SUPPORT_NAME_ONLY(id)
  return get_id(dv_ind, amd::smi::kDevSubSysDevID, id);
}

rsmi_status_t
rsmi_dev_vendor_id_get(uint32_t dv_ind, uint16_t *id) {
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  CHK_SUPPORT_NAME_ONLY(id)
  return get_id(dv_ind, amd::smi::kDevVendorID, id);
}

rsmi_status_t
rsmi_dev_subsystem_vendor_id_get(uint32_t dv_ind, uint16_t *id) {
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  CHK_SUPPORT_NAME_ONLY(id)
  return get_id(dv_ind, amd::smi::kDevSubSysVendorID, id);
}

rsmi_status_t
rsmi_dev_perf_level_get(uint32_t dv_ind, rsmi_dev_perf_level_t *perf) {
  TRY
  std::string val_str;
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  CHK_SUPPORT_NAME_ONLY(perf)
  DEVICE_MUTEX

  rsmi_status_t ret = get_dev_value_str(amd::smi::kDevPerfLevel, dv_ind,
                                                                    &val_str);
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  *perf = amd::smi::Device::perfLvlStrToEnum(val_str);

  return ret;
  CATCH
}

static rsmi_status_t
set_dev_range(uint32_t dv_ind, std::string range) {
  GET_DEV_FROM_INDX

  int ret = dev->writeDevInfo(amd::smi::kDevPowerODVoltage, range);
  return amd::smi::ErrnoToRsmiStatus(ret);
}

rsmi_status_t
rsmi_perf_determinism_mode_set(uint32_t dv_ind, uint64_t clkvalue) {
  TRY
  DEVICE_MUTEX
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << " | ======= start =======";
  LOG_TRACE(ss);

  // Set perf. level to performance determinism so that we can then set the power profile
  rsmi_status_t ret = rsmi_dev_perf_level_set_v1(dv_ind,
                                          RSMI_DEV_PERF_LEVEL_DETERMINISM);
  if (ret != RSMI_STATUS_SUCCESS) {
      return ret;
  }

  // For clock frequency setting, enter a new value by writing a string that
  // contains "s index clock" to the file. The index should be 1 to set maximum
  // clock. E.g., "s 1 500" will update maximum sclk to be 500 MHz.

  std::string sysvalue = "s";
  sysvalue += ' ' + std::to_string(RSMI_FREQ_IND_MAX);
  sysvalue += ' ' + std::to_string(clkvalue);
  sysvalue += '\n';
  ret = set_dev_range(dv_ind, sysvalue);
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }
  ret = set_dev_range(dv_ind, "c");
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  return RSMI_STATUS_SUCCESS;
  CATCH
}


rsmi_status_t
rsmi_dev_overdrive_level_get(uint32_t dv_ind, uint32_t *od) {
  TRY
  std::string val_str;
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  CHK_SUPPORT_NAME_ONLY(od)
  DEVICE_MUTEX

  rsmi_status_t ret = get_dev_value_str(amd::smi::kDevOverDriveLevel, dv_ind,
                                                                    &val_str);
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  errno = 0;
  uint64_t val_ul = strtoul(val_str.c_str(), nullptr, 10);

  if (val_ul > 0xFFFFFFFF) {
    return RSMI_STATUS_UNEXPECTED_SIZE;
  }

  *od = static_cast<uint32_t>(val_ul);
  assert(errno == 0);

  return RSMI_STATUS_SUCCESS;
  CATCH
}

rsmi_status_t
rsmi_dev_mem_overdrive_level_get(uint32_t dv_ind, uint32_t *od) {
  TRY
  std::string val_str;
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  CHK_SUPPORT_NAME_ONLY(od)
  DEVICE_MUTEX

  rsmi_status_t ret = get_dev_value_str(amd::smi::kDevMemOverDriveLevel, dv_ind,
                                                                    &val_str);
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  errno = 0;
  uint64_t val_ul = strtoul(val_str.c_str(), nullptr, 10);

  if (val_ul > 0xFFFFFFFF) {
    return RSMI_STATUS_UNEXPECTED_SIZE;
  }

  *od = static_cast<uint32_t>(val_ul);
  assert(errno == 0);

  return RSMI_STATUS_SUCCESS;
  CATCH
}

rsmi_status_t
rsmi_dev_overdrive_level_set(uint32_t dv_ind, uint32_t od) {
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  return rsmi_dev_overdrive_level_set_v1(static_cast<uint32_t>(dv_ind), od);
}

rsmi_status_t
rsmi_dev_overdrive_level_set_v1(uint32_t dv_ind, uint32_t od) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  REQUIRE_ROOT_ACCESS

  if (od > kMaxOverdriveLevel) {
    return RSMI_STATUS_INVALID_ARGS;
  }
  DEVICE_MUTEX
  return set_dev_value(amd::smi::kDevOverDriveLevel, dv_ind, od);
  CATCH
}

rsmi_status_t
rsmi_dev_perf_level_set(uint32_t dv_ind, rsmi_dev_perf_level_t perf_level) {
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  return rsmi_dev_perf_level_set_v1(dv_ind, perf_level);
}

rsmi_status_t
rsmi_dev_perf_level_set_v1(uint32_t dv_ind, rsmi_dev_perf_level_t perf_level) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  REQUIRE_ROOT_ACCESS

  if (perf_level > RSMI_DEV_PERF_LEVEL_LAST) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  DEVICE_MUTEX
  return set_dev_value(amd::smi::kDevPerfLevel, dv_ind, perf_level);
  CATCH
}


static rsmi_status_t get_frequencies(amd::smi::DevInfoTypes type, rsmi_clk_type_t clk_type,
            uint32_t dv_ind, rsmi_frequencies_t *f, uint32_t *lanes = nullptr) {
  TRY
  std::vector<std::string> val_vec;
  rsmi_status_t ret;

  if (f == nullptr) {
    return RSMI_STATUS_INVALID_ARGS;
  }
  memset(f, 0, sizeof(rsmi_frequencies_t));
  f->current=0;

  ret = GetDevValueVec(type, dv_ind, &val_vec);
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }
  assert(val_vec.size() <= RSMI_MAX_NUM_FREQUENCIES);

  if (val_vec.empty()) {
    return RSMI_STATUS_NOT_YET_IMPLEMENTED;
  }

  f->num_supported = static_cast<uint32_t>(val_vec.size());
  f->current = RSMI_MAX_NUM_FREQUENCIES + 1;  // init to an invalid value

  // Deep Sleep frequency is only supported by some GPUs
  // It is indicated by letter 'S' instead of the index number
  f->has_deep_sleep = (val_vec[0][0] == 'S');

  bool current = false;

  for (uint32_t i = 0; i < f->num_supported; ++i) {
    f->frequency[i] = freq_string_to_int(val_vec, &current, lanes, i);

    // Our assumption is that frequencies are read in from lowest to highest.
    // Check that that is true.
    if (i > 0) {
      if (f->frequency[i] < f->frequency[i-1]) {
        std::string sysvalue;
        sysvalue += kClkStateMap.find(clk_type)->second;
        sysvalue += " Current Value";
        sysvalue += ' ' + std::to_string(f->frequency[i]);
        sysvalue += " Previous Value";
        sysvalue += ' ' + std::to_string(f->frequency[i-1]);
        DEBUG_LOG("Frequencies are not read from lowest to highest. ", sysvalue);
      }
    }
    if (current) {
      // set the current frequency
      if (f->current != RSMI_MAX_NUM_FREQUENCIES + 1) {
        std::string sysvalue;
        sysvalue += kClkStateMap.find(clk_type)->second;
        sysvalue += " Current Value";
        sysvalue += ' ' + std::to_string(f->frequency[i]);
        sysvalue += " Previous Value";
        sysvalue += ' ' + std::to_string(f->frequency[f->current]);
        DEBUG_LOG("More than one current clock. ", sysvalue);
      } else {
          f->current = i;
      }
    }
  }

  // Some older drivers will not have the current frequency set
  // assert(f->current < f->num_supported);
  if (f->current >= f->num_supported) {
      f->current = -1;
      return RSMI_STATUS_UNEXPECTED_DATA;
  }

  return RSMI_STATUS_SUCCESS;
  CATCH
}

static rsmi_status_t get_power_profiles(uint32_t dv_ind,
                                        rsmi_power_profile_status_t *p,
               std::map<rsmi_power_profile_preset_masks_t, uint32_t> *ind_map) {
  TRY
  std::vector<std::string> val_vec;
  rsmi_status_t ret;

  if (p == nullptr) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  ret = GetDevValueVec(amd::smi::kDevPowerProfileMode, dv_ind, &val_vec);
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }
  assert(val_vec.size() <= RSMI_MAX_NUM_POWER_PROFILES);
  if (val_vec.size() > RSMI_MAX_NUM_POWER_PROFILES + 1 || val_vec.empty()) {
    return RSMI_STATUS_UNEXPECTED_SIZE;
  }
  // -1 for the header line, below
  p->num_profiles = static_cast<uint32_t>(val_vec.size() - 1);
  bool current = false;
  p->current = RSMI_PWR_PROF_PRST_INVALID;  // init to an invalid value
  p->available_profiles = 0;

  rsmi_power_profile_preset_masks_t prof;
  uint32_t prof_ind;

  for (uint32_t i = 1; i < val_vec.size(); ++i) {
    prof = power_prof_string_to_int(val_vec[i], &current, &prof_ind);

    if (prof == RSMI_PWR_PROF_PRST_INVALID) {
      continue;
    }

    if (ind_map != nullptr) {
      (*ind_map)[prof] = prof_ind;
    }

    p->available_profiles |= prof;
    if (current) {
      // Should only be 1 current profile
      assert(p->current == RSMI_PWR_PROF_PRST_INVALID);
      p->current = prof;
    }
  }

  assert(p->current != RSMI_PWR_PROF_PRST_INVALID);
  return RSMI_STATUS_SUCCESS;
  CATCH
}

/* We expect the pp_od_clk_voltage file to look like either of the two
formats shown below. Some of the newer ASICs will most likely have the
new format.

Old Format:
OD_SCLK:
0:        872Mhz
1:       1837Mhz
OD_MCLK:
1:       1000Mhz
OD_VDDC_CURVE:
0:        872Mhz        736mV
1:       1354Mhz        860mV
2:       1837Mhz       1186mV
OD_RANGE:
SCLK:     872Mhz       1900Mhz
MCLK:     168Mhz       1200Mhz
VDDC_CURVE_SCLK[0]:     872Mhz       1900Mhz
VDDC_CURVE_VOLT[0]:     737mV        1137mV
VDDC_CURVE_SCLK[1]:     872Mhz       1900Mhz
VDDC_CURVE_VOLT[1]:     737mV        1137mV
VDDC_CURVE_SCLK[2]:     872Mhz       1900Mhz
VDDC_CURVE_VOLT[2]:     737mV        1137mV

New Format:
GFXCLK:
0: 500Mhz
1: 800Mhz *
2: 1275Mhz
MCLK:
0: 400Mhz
1: 700Mhz
2: 1200Mhz
3: 1600Mhz *

For the new format, GFXCLK field will show min and max values(0/1). If the current
frequency in neither min/max but lies within the range, this is indicated by
an additional value followed by * at index 1 and max value at index 2.
*/
constexpr uint32_t kOD_SCLK_label_array_index = 0;
constexpr uint32_t kOD_MCLK_label_array_index =
                                            kOD_SCLK_label_array_index + 3;
constexpr uint32_t kOD_VDDC_CURVE_label_array_index =
                                            kOD_MCLK_label_array_index + 2;
constexpr uint32_t kOD_OD_RANGE_label_array_index =
                                      kOD_VDDC_CURVE_label_array_index + 4;
constexpr uint32_t kOD_VDDC_CURVE_start_index =
                                           kOD_OD_RANGE_label_array_index + 3;
// constexpr uint32_t kOD_VDDC_CURVE_num_lines =
//                                             kOD_VDDC_CURVE_start_index + 4;

static rsmi_status_t get_od_clk_volt_info(uint32_t dv_ind,
                                                  rsmi_od_volt_freq_data_t *p) {
  TRY
  std::vector<std::string> val_vec;
  rsmi_status_t ret;

  assert(p != nullptr);
  if (p == nullptr) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  ret = GetDevValueVec(amd::smi::kDevPowerODVoltage, dv_ind, &val_vec);
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  // This is a work-around to handle systems where kDevPowerODVoltage is not
  // fully supported yet.
  if (val_vec.size() < 2) {
    return RSMI_STATUS_NOT_YET_IMPLEMENTED;
  }

  assert(val_vec[kOD_SCLK_label_array_index] == "OD_SCLK:" ||
                            val_vec[kOD_SCLK_label_array_index] == "GFXCLK:");
  if ((val_vec[kOD_SCLK_label_array_index] != "OD_SCLK:") &&
                            (val_vec[kOD_SCLK_label_array_index] != "GFXCLK:")) {
    return RSMI_STATUS_UNEXPECTED_DATA;
  }

  // find last_item but skip empty lines
  int last_item = val_vec.size()-1;
  while (val_vec[last_item].empty() || val_vec[last_item][0] == 0)
      last_item--;

  p->curr_sclk_range.lower_bound = freq_string_to_int(val_vec, nullptr,
                                     nullptr, kOD_SCLK_label_array_index + 1);
  p->curr_sclk_range.upper_bound = freq_string_to_int(val_vec, nullptr,
                                     nullptr, kOD_SCLK_label_array_index + 2);

  if (val_vec.size() < (kOD_MCLK_label_array_index + 1)) {
    return RSMI_STATUS_UNEXPECTED_SIZE;
  }
  // The condition below checks if it is the old style or new style format.
  if (val_vec[kOD_MCLK_label_array_index] == "OD_MCLK:") {
    p->curr_mclk_range.lower_bound = 0;
    p->curr_mclk_range.upper_bound = freq_string_to_int(val_vec, nullptr,
                                   nullptr, kOD_MCLK_label_array_index + 1);
  } else if (val_vec[kOD_MCLK_label_array_index] == "MCLK:") {
    p->curr_mclk_range.lower_bound = freq_string_to_int(val_vec, nullptr,
                                 nullptr, kOD_MCLK_label_array_index + 1);
    // the upper memory frequency is the last
    p->curr_mclk_range.upper_bound = freq_string_to_int(val_vec, nullptr,
                                 nullptr, last_item);
    return RSMI_STATUS_SUCCESS;
  } else {
    if (val_vec.size() < (kOD_MCLK_label_array_index + 3)) {
      return RSMI_STATUS_UNEXPECTED_SIZE;
    }
    if (val_vec[kOD_MCLK_label_array_index + 1] == "MCLK:") {
      p->curr_sclk_range.upper_bound = freq_string_to_int(val_vec, nullptr,
                                   nullptr, kOD_SCLK_label_array_index + 3);
      p->curr_mclk_range.lower_bound = freq_string_to_int(val_vec, nullptr,
                                   nullptr, kOD_MCLK_label_array_index + 2);
      // the upper memory frequency is the last
      p->curr_mclk_range.upper_bound = freq_string_to_int(val_vec, nullptr,
                                   nullptr, last_item);
      return RSMI_STATUS_SUCCESS;
    }
    return RSMI_STATUS_NOT_YET_IMPLEMENTED;
  }

  if (val_vec.size() < kOD_VDDC_CURVE_label_array_index) {
    return RSMI_STATUS_UNEXPECTED_SIZE;
  }
  assert(val_vec[kOD_VDDC_CURVE_label_array_index] == "OD_VDDC_CURVE:");
  if (val_vec[kOD_VDDC_CURVE_label_array_index] != "OD_VDDC_CURVE:") {
    return RSMI_STATUS_UNEXPECTED_DATA;
  }

  uint32_t tmp = kOD_VDDC_CURVE_label_array_index + 1;
  if (val_vec.size() < (tmp + RSMI_NUM_VOLTAGE_CURVE_POINTS)) {
    return RSMI_STATUS_UNEXPECTED_SIZE;
  }
  for (uint32_t i = 0; i < RSMI_NUM_VOLTAGE_CURVE_POINTS; ++i) {
    freq_volt_string_to_point(val_vec[tmp + i], &(p->curve.vc_points[i]));
  }

  if (val_vec.size() < (kOD_OD_RANGE_label_array_index + 2)) {
    return RSMI_STATUS_UNEXPECTED_SIZE;
  }
  assert(val_vec[kOD_OD_RANGE_label_array_index] == "OD_RANGE:");
  if (val_vec[kOD_OD_RANGE_label_array_index] != "OD_RANGE:") {
    return RSMI_STATUS_UNEXPECTED_DATA;
  }

  od_value_pair_str_to_range(val_vec[kOD_OD_RANGE_label_array_index + 1],
                                                      &(p->sclk_freq_limits));
  od_value_pair_str_to_range(val_vec[kOD_OD_RANGE_label_array_index + 2],
                                                      &(p->mclk_freq_limits));

  assert((val_vec.size() - kOD_VDDC_CURVE_start_index)%2 == 0);
  if ((val_vec.size() - kOD_VDDC_CURVE_start_index)%2 != 0) {
    return RSMI_STATUS_UNEXPECTED_SIZE;
  }

  p->num_regions =
     static_cast<uint32_t>((val_vec.size() - kOD_VDDC_CURVE_start_index) / 2);

  return RSMI_STATUS_SUCCESS;
  CATCH
}

rsmi_status_t rsmi_dev_clk_extremum_set(uint32_t dv_ind, rsmi_freq_ind_t level,
                                        uint64_t clkvalue,
                                        rsmi_clk_type_t clkType) {
 TRY
  rsmi_status_t ret;
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  if (clkType != RSMI_CLK_TYPE_SYS && clkType != RSMI_CLK_TYPE_MEM) {
    return RSMI_STATUS_INVALID_ARGS;
  }
  if (level != RSMI_FREQ_IND_MIN && level != RSMI_FREQ_IND_MAX) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  std::map<rsmi_clk_type_t, std::string> clk_char_map = {
    {RSMI_CLK_TYPE_SYS, "s"},
    {RSMI_CLK_TYPE_MEM, "m"},
  };
  DEVICE_MUTEX

  // Set perf. level to manual so that we can then set the power profile
  ret = rsmi_dev_perf_level_set_v1(dv_ind, RSMI_DEV_PERF_LEVEL_MANUAL);
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  // For clock frequency setting, enter a new value by writing a string that
  // contains "s/m index clock" to the file. The index should be 0 if to set
  // minimum clock. And 1 if to set maximum clock. E.g., "s 0 500" will update
  // minimum sclk to be 500 MHz. "m 1 800" will update maximum mclk to 800Mhz.

  std::string sysvalue = clk_char_map[clkType];
  sysvalue += ' ' + std::to_string(level);
  sysvalue += ' ' + std::to_string(clkvalue);
  sysvalue += '\n';

  ret = set_dev_range(dv_ind, sysvalue);
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  ret = set_dev_range(dv_ind, "c");
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  return RSMI_STATUS_SUCCESS;
  CATCH
}

rsmi_status_t rsmi_dev_clk_range_set(uint32_t dv_ind, uint64_t minclkvalue,
                                        uint64_t maxclkvalue,
                                        rsmi_clk_type_t clkType) {
  TRY
  rsmi_status_t ret;
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  assert(minclkvalue < maxclkvalue);
  std::string min_sysvalue;
  std::string max_sysvalue;
  std::map<rsmi_clk_type_t, std::string> clk_char_map = {
    {RSMI_CLK_TYPE_SYS, "s"},
    {RSMI_CLK_TYPE_MEM, "m"},
  };
  DEVICE_MUTEX
  assert(clkType == RSMI_CLK_TYPE_SYS || clkType == RSMI_CLK_TYPE_MEM);

  // Set perf. level to manual so that we can then set the power profile
  ret = rsmi_dev_perf_level_set_v1(dv_ind, RSMI_DEV_PERF_LEVEL_MANUAL);
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  // For clock frequency setting, enter a new value by writing a string that
  // contains "s/m index clock" to the file. The index should be 0 if to set
  // minimum clock. And 1 if to set maximum clock. E.g., "s 0 500" will update
  // minimum sclk to be 500 MHz. "m 1 800" will update maximum mclk to 800Mhz.

  min_sysvalue = clk_char_map[clkType];
  min_sysvalue += ' ' + std::to_string(RSMI_FREQ_IND_MIN);
  min_sysvalue += ' ' + std::to_string(minclkvalue);
  min_sysvalue += '\n';
  max_sysvalue = clk_char_map[clkType];
  max_sysvalue += ' ' + std::to_string(RSMI_FREQ_IND_MAX);
  max_sysvalue += ' ' + std::to_string(maxclkvalue);
  max_sysvalue += '\n';

  ret = set_dev_range(dv_ind, min_sysvalue);
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }
  ret = set_dev_range(dv_ind, max_sysvalue);
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }
  ret = set_dev_range(dv_ind, "c");
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  return RSMI_STATUS_SUCCESS;
  CATCH
}

rsmi_status_t rsmi_dev_od_clk_info_set(uint32_t dv_ind, rsmi_freq_ind_t level,
                                        uint64_t clkvalue,
                                        rsmi_clk_type_t clkType) {
  TRY
  rsmi_status_t ret;
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  std::string sysvalue;
  std::map<rsmi_clk_type_t, std::string> clk_char_map = {
    {RSMI_CLK_TYPE_SYS, "s"},
    {RSMI_CLK_TYPE_MEM, "m"},
  };
  DEVICE_MUTEX

  // Set perf. level to manual so that we can then set the power profile
  ret = rsmi_dev_perf_level_set_v1(dv_ind, RSMI_DEV_PERF_LEVEL_MANUAL);
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  // For clock frequency setting, enter a new value by writing a string that
  // contains "s/m index clock" to the file. The index should be 0 if to set
  // minimum clock. And 1 if to set maximum clock. E.g., "s 0 500" will update
  // minimum sclk to be 500 MHz. "m 1 800" will update maximum mclk to 800Mhz.

  switch (clkType) {
    case RSMI_CLK_TYPE_SYS:
    case RSMI_CLK_TYPE_MEM:
      sysvalue = clk_char_map[clkType];
      sysvalue += ' ' + std::to_string(level);
      sysvalue += ' ' + std::to_string(clkvalue);
      sysvalue += '\n';
      break;

    default:
      return RSMI_STATUS_INVALID_ARGS;
  }
  ret = set_dev_range(dv_ind, sysvalue);
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }
  ret = set_dev_range(dv_ind, "c");
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  return RSMI_STATUS_SUCCESS;
  CATCH
}


rsmi_status_t rsmi_dev_od_volt_info_set(uint32_t dv_ind, uint32_t vpoint,
                                      uint64_t clkvalue, uint64_t voltvalue) {
  TRY
  rsmi_status_t ret;
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  DEVICE_MUTEX

  // Set perf. level to manual so that we can then set the power profile
  ret = rsmi_dev_perf_level_set_v1(dv_ind, RSMI_DEV_PERF_LEVEL_MANUAL);
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  // For sclk voltage curve, enter the new values by writing a string that
  // contains "vc point clock voltage" to the file. The points are indexed
  // by 0, 1 and 2. E.g., "vc 0 300 600" will update point1 with clock set
  // as 300Mhz and voltage as 600mV. "vc 2 1000 1000" will update point3
  // with clock set as 1000Mhz and voltage 1000mV.

  std::string sysvalue = "vc";
  sysvalue += ' ' + std::to_string(vpoint);
  sysvalue += ' ' + std::to_string(clkvalue);
  sysvalue += ' ' + std::to_string(voltvalue);
  sysvalue += '\n';
  ret = set_dev_range(dv_ind, sysvalue);
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }
  ret = set_dev_range(dv_ind, "c");
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  return RSMI_STATUS_SUCCESS;
  CATCH
}


static void get_vc_region(uint32_t start_ind,
                std::vector<std::string> *val_vec, rsmi_freq_volt_region_t *p) {
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << " | ======= start =======";
  LOG_TRACE(ss);
  assert(p != nullptr);
  assert(val_vec != nullptr);
  THROW_IF_NULLPTR_DEREF(p)
  THROW_IF_NULLPTR_DEREF(val_vec)

  // There must be at least 1 region to read in
  assert(val_vec->size() >= kOD_OD_RANGE_label_array_index + 2);
  assert((*val_vec)[kOD_OD_RANGE_label_array_index] == "OD_RANGE:");
  if ((val_vec->size() < kOD_OD_RANGE_label_array_index + 2)  ||
      ((*val_vec)[kOD_OD_RANGE_label_array_index] != "OD_RANGE:") ) {
    ss << __PRETTY_FUNCTION__ << " | ======= end ======= | returning "
       << getRSMIStatusString(RSMI_STATUS_UNEXPECTED_DATA);
    LOG_TRACE(ss);
    throw amd::smi::rsmi_exception(RSMI_STATUS_UNEXPECTED_DATA, __FUNCTION__);
  }
  od_value_pair_str_to_range((*val_vec)[start_ind], &p->freq_range);
  od_value_pair_str_to_range((*val_vec)[start_ind + 1], &p->volt_range);
}

/*
 * num_regions [inout] on calling, the number of regions requested to be read
 * in. At completion, the number of regions actually read in
 *
 * p [inout] point to pre-allocated memory where function will write region
 * values. Caller must make sure there is enough space for at least
 * *num_regions regions.
 */
static rsmi_status_t get_od_clk_volt_curve_regions(uint32_t dv_ind,
                            uint32_t *num_regions, rsmi_freq_volt_region_t *p) {
  TRY
  std::vector<std::string> val_vec;
  rsmi_status_t ret;
  std::ostringstream ss;

  assert(num_regions != nullptr);
  assert(p != nullptr);
  THROW_IF_NULLPTR_DEREF(p)
  THROW_IF_NULLPTR_DEREF(num_regions)

  ret = GetDevValueVec(amd::smi::kDevPowerODVoltage, dv_ind, &val_vec);
  if (ret != RSMI_STATUS_SUCCESS) {
    ss << __PRETTY_FUNCTION__
       << " | Issue: could not retreive kDevPowerODVoltage" << "; returning "
       << getRSMIStatusString(ret);
    LOG_ERROR(ss);
    return ret;
  }

  // This is a work-around to handle systems where kDevPowerODVoltage is not
  // fully supported yet.
  if (val_vec.size() < 2) {
    ss << __PRETTY_FUNCTION__
       << " | Issue: val_vec.size() < 2" << "; returning "
       << getRSMIStatusString(RSMI_STATUS_NOT_YET_IMPLEMENTED);
    LOG_ERROR(ss);
    return RSMI_STATUS_NOT_YET_IMPLEMENTED;
  }

  uint32_t val_vec_size = static_cast<uint32_t>(val_vec.size());
  assert((val_vec_size - kOD_VDDC_CURVE_start_index) > 0);
  assert((val_vec_size - kOD_VDDC_CURVE_start_index)%2 == 0);

  ss << __PRETTY_FUNCTION__
     << " | val_vec_size = " << std::dec
     << val_vec_size
     << " | kOD_VDDC_CURVE_start_index = " << kOD_VDDC_CURVE_start_index;
  LOG_DEBUG(ss);
  if (((val_vec_size - kOD_VDDC_CURVE_start_index) <= 0)  ||
      (((val_vec_size - kOD_VDDC_CURVE_start_index)%2 != 0))) {
    ss << __PRETTY_FUNCTION__ << " | Issue: od vdd curve returned unexpected "
       << "data" << "; returning "
       << getRSMIStatusString(RSMI_STATUS_UNEXPECTED_SIZE);
    LOG_ERROR(ss);
    throw amd::smi::rsmi_exception(RSMI_STATUS_UNEXPECTED_SIZE, __FUNCTION__);
  }

  *num_regions = std::min((val_vec_size - kOD_VDDC_CURVE_start_index) / 2,
                                                                *num_regions);

  for (uint32_t i=0; i < *num_regions; ++i) {
    get_vc_region(kOD_VDDC_CURVE_start_index + i*2, &val_vec, p + i);
  }

  return RSMI_STATUS_SUCCESS;
  CATCH
}

static rsmi_status_t set_power_profile(uint32_t dv_ind,
                                    rsmi_power_profile_preset_masks_t profile) {
  TRY

  rsmi_status_t ret;
  rsmi_power_profile_status_t avail_profiles =
                                           {0, RSMI_PWR_PROF_PRST_INVALID, 0};

  // Determine if the provided profile is valid
  if (!is_power_of_2(profile)) {
    return RSMI_STATUS_INPUT_OUT_OF_BOUNDS;
  }

  std::map<rsmi_power_profile_preset_masks_t, uint32_t> ind_map;
  ret = get_power_profiles(dv_ind, &avail_profiles, &ind_map);

  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  if (!(profile & avail_profiles.available_profiles)) {
    return RSMI_STATUS_INPUT_OUT_OF_BOUNDS;
  }
  assert(ind_map.find(profile) != ind_map.end());

  // Set perf. level to manual so that we can then set the power profile
  ret = rsmi_dev_perf_level_set_v1(dv_ind, RSMI_DEV_PERF_LEVEL_MANUAL);
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  // Write the new profile
  ret = set_dev_value(amd::smi::kDevPowerProfileMode, dv_ind,
                                                            ind_map[profile]);

  return ret;
  CATCH
}

static rsmi_status_t topo_get_numa_node_number(uint32_t dv_ind,
                     uint32_t *numa_node_number) {
  TRY

  GET_DEV_AND_KFDNODE_FROM_INDX

  *numa_node_number = kfd_node->numa_node_number();

  return RSMI_STATUS_SUCCESS;
  CATCH
}

static rsmi_status_t topo_get_numa_node_weight(uint32_t dv_ind,
                     uint64_t *weight) {
  TRY

  GET_DEV_AND_KFDNODE_FROM_INDX

  *weight = kfd_node->numa_node_weight();

  return RSMI_STATUS_SUCCESS;
  CATCH
}

rsmi_status_t
rsmi_dev_gpu_clk_freq_get(uint32_t dv_ind, rsmi_clk_type_t clk_type,
                                                        rsmi_frequencies_t *f) {
  TRY
  amd::smi::DevInfoTypes dev_type;
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  CHK_SUPPORT_VAR(f, clk_type)

  const auto & clk_type_it = kClkTypeMap.find(clk_type);
  if (clk_type_it != kClkTypeMap.end()) {
    dev_type = clk_type_it->second;
  } else {
    return RSMI_STATUS_INVALID_ARGS;
  }

  DEVICE_MUTEX

  return get_frequencies(dev_type, clk_type, dv_ind, f);

  CATCH
}

rsmi_status_t
rsmi_dev_firmware_version_get(uint32_t dv_ind, rsmi_fw_block_t block,
                                                       uint64_t *fw_version) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  CHK_SUPPORT_VAR(fw_version, block)

  std::string val_str;
  amd::smi::DevInfoTypes dev_type;

  static const std::map<rsmi_fw_block_t, amd::smi::DevInfoTypes> kFWBlockTypeMap = {
    { RSMI_FW_BLOCK_ASD, amd::smi::kDevFwVersionAsd },
    { RSMI_FW_BLOCK_CE, amd::smi::kDevFwVersionCe },
    { RSMI_FW_BLOCK_DMCU, amd::smi::kDevFwVersionDmcu },
    { RSMI_FW_BLOCK_MC, amd::smi::kDevFwVersionMc },
    { RSMI_FW_BLOCK_ME, amd::smi::kDevFwVersionMe },
    { RSMI_FW_BLOCK_MEC, amd::smi::kDevFwVersionMec },
    { RSMI_FW_BLOCK_MEC2, amd::smi::kDevFwVersionMec2 },
    { RSMI_FW_BLOCK_MES, amd::smi::kDevFwVersionMes },
    { RSMI_FW_BLOCK_MES_KIQ, amd::smi::kDevFwVersionMesKiq },
    { RSMI_FW_BLOCK_PFP, amd::smi::kDevFwVersionPfp },
    { RSMI_FW_BLOCK_RLC, amd::smi::kDevFwVersionRlc },
    { RSMI_FW_BLOCK_RLC_SRLC, amd::smi::kDevFwVersionRlcSrlc },
    { RSMI_FW_BLOCK_RLC_SRLG, amd::smi::kDevFwVersionRlcSrlg },
    { RSMI_FW_BLOCK_RLC_SRLS, amd::smi::kDevFwVersionRlcSrls },
    { RSMI_FW_BLOCK_SDMA, amd::smi::kDevFwVersionSdma },
    { RSMI_FW_BLOCK_SDMA2, amd::smi::kDevFwVersionSdma2 },
    { RSMI_FW_BLOCK_SMC, amd::smi::kDevFwVersionSmc },
    { RSMI_FW_BLOCK_SOS, amd::smi::kDevFwVersionSos },
    { RSMI_FW_BLOCK_TA_RAS, amd::smi::kDevFwVersionTaRas },
    { RSMI_FW_BLOCK_TA_XGMI, amd::smi::kDevFwVersionTaXgmi },
    { RSMI_FW_BLOCK_UVD, amd::smi::kDevFwVersionUvd },
    { RSMI_FW_BLOCK_VCE, amd::smi::kDevFwVersionVce },
    { RSMI_FW_BLOCK_VCN, amd::smi::kDevFwVersionVcn },
  };

  const auto & dev_type_it = kFWBlockTypeMap.find(block);
  if (dev_type_it != kFWBlockTypeMap.end()) {
    dev_type = dev_type_it->second;
  } else {
    return RSMI_STATUS_INVALID_ARGS;
  }

  DEVICE_MUTEX
  return get_dev_value_int(dev_type, dv_ind, fw_version);
  CATCH
}

static std::string bitfield_to_freq_string(uint64_t bitf,
                                                     uint32_t num_supported) {
  std::string bf_str;
  std::bitset<RSMI_MAX_NUM_FREQUENCIES> bs(bitf);

  if (num_supported > RSMI_MAX_NUM_FREQUENCIES) {
    throw amd::smi::rsmi_exception(RSMI_STATUS_INVALID_ARGS, __FUNCTION__);
  }

  for (uint32_t i = 0; i < num_supported; ++i) {
    if (bs[i]) {
      bf_str += std::to_string(i);
      bf_str += " ";
    }
  }
  return bf_str;
}

rsmi_status_t
rsmi_dev_gpu_clk_freq_set(uint32_t dv_ind,
                              rsmi_clk_type_t clk_type, uint64_t freq_bitmask) {
  rsmi_status_t ret;
  rsmi_frequencies_t freqs;

  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << " | ======= start =======";
  LOG_TRACE(ss);
  REQUIRE_ROOT_ACCESS
  DEVICE_MUTEX

  if (clk_type > RSMI_CLK_TYPE_LAST) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  ret = rsmi_dev_gpu_clk_freq_get(dv_ind, clk_type, &freqs);

  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  assert(freqs.num_supported <= RSMI_MAX_NUM_FREQUENCIES);
  if (freqs.num_supported > RSMI_MAX_NUM_FREQUENCIES) {
    return RSMI_STATUS_UNEXPECTED_SIZE;
  }

  amd::smi::RocmSMI& smi = amd::smi::RocmSMI::getInstance();

  // Above call to rsmi_dev_get_gpu_clk_freq should have emitted an error if
  // assert below is not true
  assert(dv_ind < smi.devices().size());

  std::string freq_enable_str =
                   bitfield_to_freq_string(freq_bitmask, freqs.num_supported);


  std::shared_ptr<amd::smi::Device> dev = smi.devices()[dv_ind];
  assert(dev != nullptr);

  ret = rsmi_dev_perf_level_set_v1(dv_ind, RSMI_DEV_PERF_LEVEL_MANUAL);
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  int ret_i;
  amd::smi::DevInfoTypes dev_type;

  const auto & clk_type_it = kClkTypeMap.find(clk_type);
  if (clk_type_it != kClkTypeMap.end()) {
    dev_type = clk_type_it->second;
  } else {
    return RSMI_STATUS_INVALID_ARGS;
  }

  ret_i = dev->writeDevInfo(dev_type, freq_enable_str);
  return amd::smi::ErrnoToRsmiStatus(ret_i);

  CATCH
}
static std::vector<std::string> pci_name_files = {
  "/usr/share/misc/pci.ids",
  "/usr/share/hwdata/pci.ids",
  "/usr/share/pci.ids",
  "/var/lib/pciutils/pci.ids"
};

enum eNameStrType {
  NAME_STR_VENDOR = 0,
  NAME_STR_DEVICE,
  NAME_STR_SUBSYS
};

std::map<std::string, rsmi_compute_partition_type_t>
mapStringToRSMIComputePartitionTypes {
  {"CPX", RSMI_COMPUTE_PARTITION_CPX},
  {"SPX", RSMI_COMPUTE_PARTITION_SPX},
  {"DPX", RSMI_COMPUTE_PARTITION_DPX},
  {"TPX", RSMI_COMPUTE_PARTITION_TPX},
  {"QPX", RSMI_COMPUTE_PARTITION_QPX}
};

std::map<rsmi_compute_partition_type_t, std::string>
mapRSMIToStringComputePartitionTypes {
  {RSMI_COMPUTE_PARTITION_INVALID, "UNKNOWN"},
  {RSMI_COMPUTE_PARTITION_CPX, "CPX"},
  {RSMI_COMPUTE_PARTITION_SPX, "SPX"},
  {RSMI_COMPUTE_PARTITION_DPX, "DPX"},
  {RSMI_COMPUTE_PARTITION_TPX, "TPX"},
  {RSMI_COMPUTE_PARTITION_QPX, "QPX"}
};

std::map<rsmi_memory_partition_type_t, std::string>
mapRSMIToStringMemoryPartitionTypes {
  {RSMI_MEMORY_PARTITION_UNKNOWN, "UNKNOWN"},
  {RSMI_MEMORY_PARTITION_NPS1, "NPS1"},
  {RSMI_MEMORY_PARTITION_NPS2, "NPS2"},
  {RSMI_MEMORY_PARTITION_NPS4, "NPS4"},
  {RSMI_MEMORY_PARTITION_NPS8, "NPS8"}
};

std::map<std::string, rsmi_memory_partition_type_t>
mapStringToMemoryPartitionTypes {
  {"NPS1", RSMI_MEMORY_PARTITION_NPS1},
  {"NPS2", RSMI_MEMORY_PARTITION_NPS2},
  {"NPS4", RSMI_MEMORY_PARTITION_NPS4},
  {"NPS8", RSMI_MEMORY_PARTITION_NPS8}
};

static std::string
get_id_name_str_from_line(uint64_t id, std::string ln,
                                                 std::istringstream *ln_str) {
  std::string token1;
  std::string ret_str;

  assert(ln_str != nullptr);
  THROW_IF_NULLPTR_DEREF(ln_str)

  *ln_str >> token1;

  if (token1.empty()) {
    throw amd::smi::rsmi_exception(RSMI_STATUS_NO_DATA, __FUNCTION__);
  }

  if (std::stoul(token1, nullptr, 16) == id) {
    int64_t pos = ln_str->tellg();

    assert(pos >= 0);
    if (pos < 0) {
      throw amd::smi::rsmi_exception(
          RSMI_STATUS_UNEXPECTED_DATA, __FUNCTION__);
    }
    size_t s_pos = ln.find_first_not_of("\t ", static_cast<size_t>(pos));
    ret_str = ln.substr(static_cast<uint32_t>(s_pos));
  }
  return ret_str;
}

static rsmi_status_t get_backup_name(uint16_t id, char *name, size_t len) {
  std::string name_str;

  assert(name != nullptr);
  if (name == nullptr) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  name_str += "0x";

  std::stringstream strm;
  strm << std::hex << id;
  name_str += strm.str();

  name[0] = '\0';
  size_t ct = name_str.copy(name, len);

  name[std::min(len - 1, ct)] = '\0';

  if (len < (name_str.size() + 1)) {
    return RSMI_STATUS_INSUFFICIENT_SIZE;
  }
  return RSMI_STATUS_SUCCESS;
}

static rsmi_status_t get_dev_name_from_file(uint32_t dv_ind, char *name,
                                               size_t len) {
  std::string val_str;
  rsmi_status_t ret =
              get_dev_value_line(amd::smi::kDevDevProdName, dv_ind, &val_str);

  if (ret != 0) {
    return amd::smi::ErrnoToRsmiStatus(ret);
  }
  size_t ct = val_str.copy(name, len);

  name[std::min(len - 1, ct)] = '\0';

  if (len < (val_str.size() + 1)) {
    return RSMI_STATUS_INSUFFICIENT_SIZE;
  }

  return RSMI_STATUS_SUCCESS;
}

// Parse pci.ids files. Comment lines have # in first column. Otherwise,
// Syntax:
// vendor  vendor_name
//       device  device_name                             <-- single tab
//               subvendor subdevice  subsystem_name     <-- two tabs
static rsmi_status_t get_dev_name_from_id(uint32_t dv_ind, char *name,
                                               size_t len, eNameStrType typ) {
  std::string ln;
  std::string token1;
  rsmi_status_t ret;
  uint16_t device_id;
  uint16_t vendor_id;
  uint16_t subsys_vend_id;
  uint16_t subsys_id;
  bool found_device_vendor = false;
  // to match subsystem, it must match the device id at previous line
  bool found_device_id_for_subsys = false;
  std::string val_str;

  assert(name != nullptr);
  assert(len > 0);

  if (name == nullptr || len == 0) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  name[0] = '\0';

  ret = rsmi_dev_vendor_id_get(dv_ind, &vendor_id);
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  if (typ != NAME_STR_VENDOR) {
    ret = rsmi_dev_id_get(dv_ind, &device_id);
    if (ret != RSMI_STATUS_SUCCESS) {
      return ret;
    }
    if (typ != NAME_STR_DEVICE) {
      ret = rsmi_dev_subsystem_vendor_id_get(dv_ind, &subsys_vend_id);
      if (ret != RSMI_STATUS_SUCCESS) {
        return ret;
      }
      ret = rsmi_dev_subsystem_id_get(dv_ind, &subsys_id);
      if (ret != RSMI_STATUS_SUCCESS) {
        return ret;
      }
    }
  }

  for (const auto& fl : pci_name_files) {
    std::ifstream id_file_strm(fl);

    while (std::getline(id_file_strm, ln)) {
      std::istringstream ln_str(ln);
      // parse line
      if (ln[0] == '#' || ln.empty()) {
        continue;
      }

      if (ln[0] == '\t') {
        if (found_device_vendor) {
          if (ln[1] == '\t') {
            // The subsystem line, ignore a line if the device id not match
            if (typ == NAME_STR_SUBSYS && found_device_id_for_subsys) {
              val_str = get_id_name_str_from_line(subsys_vend_id, ln, &ln_str);

              if (!val_str.empty()) {
                // We've chopped the subsys_vend ID, now we need to get the
                // subsys description
                val_str = get_id_name_str_from_line(subsys_id, ln, &ln_str);

                if (!val_str.empty()) {
                  break;
                }
                val_str.clear();
              }
            }
          } else if (typ == NAME_STR_DEVICE) {  // ln[1] != '\t'
            // This is a device line
            val_str = get_id_name_str_from_line(device_id, ln, &ln_str);

            if (!val_str.empty()) {
              break;
            }
          } else if (typ == NAME_STR_SUBSYS) {
            // match the device id line
            val_str = get_id_name_str_from_line(device_id, ln, &ln_str);
            if (!val_str.empty()) {
              found_device_id_for_subsys = true;
            }
          }
        }
      } else {  // ln[0] != '\t'; Vendor line
        if (found_device_vendor) {
          assert(typ != NAME_STR_VENDOR);
          // We already found the vendor but didn't find the device or
          // subsystem we were looking for, so bail out.
          val_str.clear();

          return get_backup_name(typ == NAME_STR_DEVICE ?
                                            device_id : subsys_id, name, len);
        }

        val_str = get_id_name_str_from_line(vendor_id, ln, &ln_str);

        if (!val_str.empty()) {
          if (typ == NAME_STR_VENDOR) {
            break;
          }
          val_str.clear();
          found_device_vendor = true;
        }
      }
    }
    if (!val_str.empty()) {
      break;
    }
  }

  if (val_str.empty()) {
    return get_backup_name(vendor_id, name, len);
  }
  size_t ct = val_str.copy(name, len);

  name[std::min(len - 1, ct)] = '\0';

  if (len < (val_str.size() + 1)) {
    return RSMI_STATUS_INSUFFICIENT_SIZE;
  }

  return RSMI_STATUS_SUCCESS;
}

static rsmi_status_t
get_dev_drm_render_minor(uint32_t dv_ind, uint32_t *minor) {
  GET_DEV_FROM_INDX

  assert(minor != nullptr);
  if (minor == nullptr) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  *minor = dev->drm_render_minor();
  if (*minor)
    return RSMI_STATUS_SUCCESS;

  return RSMI_STATUS_INIT_ERROR;
}

rsmi_status_t
rsmi_dev_name_get(uint32_t dv_ind, char *name, size_t len) {
  rsmi_status_t ret;

  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  CHK_SUPPORT_NAME_ONLY(name)

  if (len == 0) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  DEVICE_MUTEX

  ret = get_dev_name_from_file(dv_ind, name, len);

  if (ret || name[0] == '\0' || !isprint(name[0]) ) {
    ret = get_dev_name_from_id(dv_ind, name, len, NAME_STR_DEVICE);
  }

  return ret;
  CATCH
}

rsmi_status_t
rsmi_dev_brand_get(uint32_t dv_ind, char *brand, uint32_t len) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  CHK_SUPPORT_NAME_ONLY(brand)
  if (len == 0) {
    return RSMI_STATUS_INVALID_ARGS;
  }
  DEVICE_MUTEX

  std::map<std::string, std::string> brand_names = {
    {"D05121", "mi25"},
    {"D05131", "mi25"},
    {"D05133", "mi25"},
    {"D05151", "mi25"},
    {"D16304", "mi50"},
    {"D16302", "mi60"}
  };
  std::map<std::string, std::string>::iterator it;
  std::string vbios_value;
  std::string sku_value;
  // Retrieve vbios and store in vbios_value string
  int ret = dev->readDevInfo(amd::smi::kDevVBiosVer, &vbios_value);
  if (ret != 0) {
    return amd::smi::ErrnoToRsmiStatus(ret);
  }
  if (vbios_value.length() == 16) {
    sku_value = vbios_value.substr(4, 6);
    // Find the brand name using sku_value
    it = brand_names.find(sku_value);
    if (it != brand_names.end()) {
      uint32_t ln = static_cast<uint32_t>(it->second.copy(brand, len));
      brand[std::min(len - 1, ln)] = '\0';

      if (len < (it->second.size() + 1)) {
        return RSMI_STATUS_INSUFFICIENT_SIZE;
      }

      return RSMI_STATUS_SUCCESS;
    }
  }
  // If there is no SKU match, return marketing name instead
  rsmi_dev_name_get(dv_ind, brand, len);
  return RSMI_STATUS_SUCCESS;
  CATCH
}

rsmi_status_t
rsmi_dev_vram_vendor_get(uint32_t dv_ind, char *brand, uint32_t len) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  CHK_SUPPORT_NAME_ONLY(brand)

  if (len == 0) {
    return RSMI_STATUS_INVALID_ARGS;
  }
  std::string val_str;
  DEVICE_MUTEX
  int ret = dev->readDevInfo(amd::smi::kDevVramVendor, &val_str);

  if (ret != 0) {
    return amd::smi::ErrnoToRsmiStatus(ret);
  }

  uint32_t ln = static_cast<uint32_t>(val_str.copy(brand, len));

  brand[std::min(len - 1, ln)] = '\0';

  if (len < (val_str.size() + 1)) {
    return RSMI_STATUS_INSUFFICIENT_SIZE;
  }
  return RSMI_STATUS_SUCCESS;

  CATCH
}

rsmi_status_t
rsmi_dev_subsystem_name_get(uint32_t dv_ind, char *name, size_t len) {
  rsmi_status_t ret;

  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  CHK_SUPPORT_NAME_ONLY(name)

  if (len == 0) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  DEVICE_MUTEX

  ret = get_dev_name_from_id(dv_ind, name, len, NAME_STR_SUBSYS);
  return ret;
  CATCH
}

rsmi_status_t
rsmi_dev_drm_render_minor_get(uint32_t dv_ind, uint32_t *minor) {
  rsmi_status_t ret;

  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  CHK_SUPPORT_NAME_ONLY(minor)

  DEVICE_MUTEX
  ret = get_dev_drm_render_minor(dv_ind, minor);
  return ret;
  CATCH
}

rsmi_status_t
rsmi_dev_vendor_name_get(uint32_t dv_ind, char *name, size_t len) {
  rsmi_status_t ret;

  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  CHK_SUPPORT_NAME_ONLY(name)

  assert(len > 0);
  if (len == 0) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  DEVICE_MUTEX
  ret = get_dev_name_from_id(dv_ind, name, len, NAME_STR_VENDOR);
  return ret;
  CATCH
}


rsmi_status_t
rsmi_dev_pci_bandwidth_get(uint32_t dv_ind, rsmi_pcie_bandwidth_t *b) {
  rsmi_status_t ret;
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  GET_DEV_AND_KFDNODE_FROM_INDX
  CHK_API_SUPPORT_ONLY((b), RSMI_DEFAULT_VARIANT, RSMI_DEFAULT_VARIANT)
  DEVICE_MUTEX
  ret = get_frequencies(amd::smi::kDevPCIEClk, RSMI_CLK_TYPE_PCIE, dv_ind,
                                        &b->transfer_rate, b->lanes);
  if (ret == RSMI_STATUS_SUCCESS) {
    return ret;
  }

  // Only fallback to gpu_metric if connecting via PCIe
  if (kfd_node->numa_node_type() != amd::smi::IOLINK_TYPE_PCIEXPRESS) {
    return ret;
  }

  rsmi_gpu_metrics_t gpu_metrics;
  ret = rsmi_dev_gpu_metrics_info_get(dv_ind, &gpu_metrics);
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  // Hardcode based on PCIe specification: https://en.wikipedia.org/wiki/PCI_Express
  const uint32_t link_width[] = {1, 2, 4, 8, 12, 16};
  const uint32_t link_speed[] = {25, 50, 80, 160};  // 0.1 Ghz
  const uint32_t WIDTH_DATA_LENGTH = sizeof(link_width)/sizeof(uint32_t);
  const uint32_t SPEED_DATA_LENGTH = sizeof(link_speed)/sizeof(uint32_t);

  // Calculate the index
  uint32_t width_index = -1;
  uint32_t speed_index = -1;
  uint32_t cur_index = 0;
  for (cur_index = 0; cur_index < WIDTH_DATA_LENGTH; cur_index++) {
    if (link_width[cur_index] == gpu_metrics.pcie_link_width) {
      width_index = cur_index;
      break;
    }
  }
  for (cur_index = 0; cur_index < SPEED_DATA_LENGTH; cur_index++) {
    if (link_speed[cur_index] == gpu_metrics.pcie_link_speed) {
      speed_index = cur_index;
      break;
    }
  }
  if (width_index == -1 || speed_index == -1) {
    return RSMI_STATUS_NOT_SUPPORTED;
  }
  // Set possible lanes and frequencies
  b->transfer_rate.num_supported = WIDTH_DATA_LENGTH * SPEED_DATA_LENGTH;
  b->transfer_rate.current = speed_index*WIDTH_DATA_LENGTH + width_index;
  for (cur_index = 0; cur_index < WIDTH_DATA_LENGTH * SPEED_DATA_LENGTH; cur_index++) {
    b->transfer_rate.frequency[cur_index] =
      static_cast<long>(link_speed[cur_index/WIDTH_DATA_LENGTH]) * 100 * 1000000L;
    b->lanes[cur_index] = link_width[cur_index % WIDTH_DATA_LENGTH];
  }
  /*
  frequency = {2500, 2500, 2500, 2500, 2500, 2500,
              5000, 5000, 5000, 5000, 5000, 5000,
              8000, 8000, 8000, 8000, 8000, 8000,
              16000, 16000, 16000, 16000, 16000, 16000};  // Mhz
  lanes = {1, 2, 4, 8, 12, 16,
              1, 2, 4, 8, 12, 16,
              1, 2, 4, 8, 12, 16,
              1, 2, 4, 8, 12, 16 };  // For each frequency
  */

  return RSMI_STATUS_SUCCESS;

  CATCH
}

rsmi_status_t
rsmi_dev_pci_bandwidth_set(uint32_t dv_ind, uint64_t bw_bitmask) {
  rsmi_status_t ret;
  rsmi_pcie_bandwidth_t bws;

  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  REQUIRE_ROOT_ACCESS
  DEVICE_MUTEX
  ret = rsmi_dev_pci_bandwidth_get(dv_ind, &bws);

  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  assert(bws.transfer_rate.num_supported <= RSMI_MAX_NUM_FREQUENCIES);

  amd::smi::RocmSMI& smi = amd::smi::RocmSMI::getInstance();

  // Above call to rsmi_dev_pci_bandwidth_get() should have emitted an error
  // if assert below is not true
  assert(dv_ind < smi.devices().size());

  std::string freq_enable_str =
         bitfield_to_freq_string(bw_bitmask, bws.transfer_rate.num_supported);

  std::shared_ptr<amd::smi::Device> dev = smi.devices()[dv_ind];
  assert(dev != nullptr);

  ret = rsmi_dev_perf_level_set_v1(dv_ind, RSMI_DEV_PERF_LEVEL_MANUAL);
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  int32_t ret_i;
  ret_i = dev->writeDevInfo(amd::smi::kDevPCIEClk, freq_enable_str);

  return amd::smi::ErrnoToRsmiStatus(ret_i);

  CATCH
}

rsmi_status_t
rsmi_dev_pci_throughput_get(uint32_t dv_ind, uint64_t *sent,
                                   uint64_t *received, uint64_t *max_pkt_sz) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  rsmi_status_t ret;
  std::string val_str;

  // We don't do CHK_SUPPORT_NAME_ONLY in this case as the user may
  // choose to have any of the inout parameters as 0. Let the return code from
  // get_dev_value_line() tell if this function is supported or not.
  // CHK_SUPPORT_NAME_ONLY(...)

  DEVICE_MUTEX

  ret = get_dev_value_line(amd::smi::kDevPCIEThruPut, dv_ind, &val_str);

  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  std::istringstream fs_rng(val_str);

  if (sent) {
    fs_rng >> *sent;
  }
  if (received) {
    fs_rng >> *received;
  }
  if (max_pkt_sz) {
    fs_rng >> *max_pkt_sz;
  }

  return RSMI_STATUS_SUCCESS;
  CATCH
}

rsmi_status_t
rsmi_dev_temp_metric_get(uint32_t dv_ind, uint32_t sensor_type,
                       rsmi_temperature_metric_t metric, int64_t *temperature) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  rsmi_status_t ret;
  amd::smi::MonitorTypes mon_type = amd::smi::kMonInvalid;
  uint16_t val_ui16;

  static const std::map<rsmi_temperature_metric_t, amd::smi::MonitorTypes>
    kMetricTypeMap = {
      { RSMI_TEMP_CURRENT, amd::smi::kMonTemp },
      { RSMI_TEMP_MAX, amd::smi::kMonTempMax },
      { RSMI_TEMP_MIN, amd::smi::kMonTempMin },
      { RSMI_TEMP_MAX_HYST, amd::smi::kMonTempMaxHyst },
      { RSMI_TEMP_MIN_HYST, amd::smi::kMonTempMinHyst },
      { RSMI_TEMP_CRITICAL, amd::smi::kMonTempCritical },
      { RSMI_TEMP_CRITICAL_HYST, amd::smi::kMonTempCriticalHyst },
      { RSMI_TEMP_EMERGENCY, amd::smi::kMonTempEmergency },
      { RSMI_TEMP_EMERGENCY_HYST, amd::smi::kMonTempEmergencyHyst },
      { RSMI_TEMP_CRIT_MIN, amd::smi::kMonTempCritMin },
      { RSMI_TEMP_CRIT_MIN_HYST, amd::smi::kMonTempCritMinHyst },
      { RSMI_TEMP_OFFSET, amd::smi::kMonTempOffset },
      { RSMI_TEMP_LOWEST, amd::smi::kMonTempLowest },
      { RSMI_TEMP_HIGHEST, amd::smi::kMonTempHighest },
  };

  const auto mon_type_it = kMetricTypeMap.find(metric);
  if (mon_type_it != kMetricTypeMap.end()) {
    mon_type = mon_type_it->second;
  }

  if (temperature == nullptr) {
    ss << __PRETTY_FUNCTION__
       << " | ======= end ======= "
       << " | Fail "
       << " | Device #: " << dv_ind
       << " | Type: " << monitorTypesToString.at(mon_type)
       << " | Cause: temperature was a null ptr reference"
       << " | Returning = "
       << getRSMIStatusString(RSMI_STATUS_INVALID_ARGS) << " |";
    LOG_ERROR(ss);
    return RSMI_STATUS_INVALID_ARGS;
  }

  // The HBM temperature is retrieved from the gpu_metrics
  if (sensor_type == RSMI_TEMP_TYPE_HBM_0 ||
      sensor_type == RSMI_TEMP_TYPE_HBM_1 ||
      sensor_type == RSMI_TEMP_TYPE_HBM_2 ||
      sensor_type == RSMI_TEMP_TYPE_HBM_3) {
    if (metric != RSMI_TEMP_CURRENT) {   // only support RSMI_TEMP_CURRENT
      ss << __PRETTY_FUNCTION__
         << " | ======= end ======= "
         << " | Fail "
         << " | Device #: " << dv_ind
         << " | Type: " << monitorTypesToString.at(mon_type)
         << " | Cause: To retrieve HBM temp, we only support metric = "
         << "RSMI_TEMP_CURRENT"
         << " | Returning = "
         << getRSMIStatusString(RSMI_STATUS_NOT_SUPPORTED) << " |";
      LOG_ERROR(ss);
      return RSMI_STATUS_NOT_SUPPORTED;
    }

    rsmi_gpu_metrics_t gpu_metrics;
    ret = rsmi_dev_gpu_metrics_info_get(dv_ind, &gpu_metrics);
    if (ret != RSMI_STATUS_SUCCESS) {
      ss << __PRETTY_FUNCTION__
         << " | ======= end ======= "
         << " | Fail "
         << " | Device #: " << dv_ind
         << " | Type: " << monitorTypesToString.at(mon_type)
         << " | Cause: rsmi_dev_gpu_metrics_info_get returned "
         << getRSMIStatusString(ret)
         << " | Returning = "
         << getRSMIStatusString(ret) << " |";
      LOG_ERROR(ss);
      return ret;
    }

    switch (sensor_type) {
      case RSMI_TEMP_TYPE_HBM_0:
        val_ui16 = gpu_metrics.temperature_hbm[0];
        break;
      case RSMI_TEMP_TYPE_HBM_1:
        val_ui16 = gpu_metrics.temperature_hbm[1];
        break;
      case RSMI_TEMP_TYPE_HBM_2:
        val_ui16 = gpu_metrics.temperature_hbm[2];
        break;
      case RSMI_TEMP_TYPE_HBM_3:
        val_ui16 = gpu_metrics.temperature_hbm[3];
        break;
      default:
        return RSMI_STATUS_INVALID_ARGS;
    }
    if (val_ui16 == UINT16_MAX) {
      ss << __PRETTY_FUNCTION__
         << " | ======= end ======= "
         << " | Fail "
         << " | Device #: " << dv_ind
         << " | Type: " << monitorTypesToString.at(mon_type)
         << " | Cause: Reached UINT16 max value, overflow"
         << " | Returning = "
         << getRSMIStatusString(RSMI_STATUS_NOT_SUPPORTED) << " |";
      LOG_ERROR(ss);
      return RSMI_STATUS_NOT_SUPPORTED;
    }

    *temperature =
      static_cast<int64_t>(val_ui16) * CENTRIGRADE_TO_MILLI_CENTIGRADE;

    ss << __PRETTY_FUNCTION__ << " | ======= end ======= "
       << " | Success "
       << " | Device #: " << dv_ind
       << " | Type: " << monitorTypesToString.at(mon_type)
       << " | Data: " << *temperature
       << " | Returning = "
       << getRSMIStatusString(RSMI_STATUS_SUCCESS) << " | ";
    LOG_INFO(ss);
    return RSMI_STATUS_SUCCESS;
  }  // end HBM temperature

  DEVICE_MUTEX

  GET_DEV_FROM_INDX

  if (dev->monitor() == nullptr) {
    ss << __PRETTY_FUNCTION__
       << " | ======= end ======= "
       << " | Fail "
       << " | Device #: " << dv_ind
       << " | Type: " << monitorTypesToString.at(mon_type)
       << " | Cause: monitor returned nullptr"
       << " | Returning = "
       << getRSMIStatusString(RSMI_STATUS_NOT_SUPPORTED) << " |";
    LOG_ERROR(ss);
    return RSMI_STATUS_NOT_SUPPORTED;
  }
  std::shared_ptr<amd::smi::Monitor> m = dev->monitor();

  // getTempSensorIndex will throw an out of range exception if sensor_type is
  // not found
  uint32_t sensor_index =
     m->getTempSensorIndex(static_cast<rsmi_temperature_type_t>(sensor_type));


  CHK_API_SUPPORT_ONLY(temperature, metric, sensor_index)

  ret = get_dev_mon_value(mon_type, dv_ind, sensor_index, temperature);
  ss << __PRETTY_FUNCTION__ << " | ======= end ======= "
     << " | Success "
     << " | Device #: " << dv_ind
     << " | Sensor_index: " << sensor_index
     << " | Type: " << monitorTypesToString.at(mon_type)
     << " | Data: " << *temperature
     << " | Returning = "
     << getRSMIStatusString(ret) << " | ";
  LOG_INFO(ss);

  return ret;
  CATCH
}

rsmi_status_t
rsmi_dev_volt_metric_get(uint32_t dv_ind, rsmi_voltage_type_t sensor_type,
                       rsmi_voltage_metric_t metric, int64_t *voltage) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  rsmi_status_t ret;
  amd::smi::MonitorTypes mon_type;

  switch (metric) {
    case RSMI_VOLT_CURRENT:
      mon_type = amd::smi::kMonVolt;
      break;
    case RSMI_VOLT_MIN:
      mon_type = amd::smi::kMonVoltMin;
      break;
    case RSMI_VOLT_MIN_CRIT:
      mon_type = amd::smi::kMonVoltMinCrit;
      break;
    case RSMI_VOLT_MAX:
      mon_type = amd::smi::kMonVoltMax;
      break;
    case RSMI_VOLT_MAX_CRIT:
      mon_type = amd::smi::kMonVoltMaxCrit;
      break;
    case RSMI_VOLT_AVERAGE:
      mon_type = amd::smi::kMonVoltAverage;
      break;
    case RSMI_VOLT_LOWEST:
      mon_type = amd::smi::kMonVoltLowest;
      break;
    case RSMI_VOLT_HIGHEST:
      mon_type = amd::smi::kMonVoltHighest;
      break;
    default:
      mon_type = amd::smi::kMonInvalid;
  }

  DEVICE_MUTEX

  GET_DEV_FROM_INDX

  if (dev->monitor() == nullptr) {
    return RSMI_STATUS_NOT_SUPPORTED;
  }
  std::shared_ptr<amd::smi::Monitor> m = dev->monitor();

  // getVoltSensorIndex will throw an out of range exception if sensor_type is
  // not found
  uint32_t sensor_index;
  try {
    sensor_index =
      m->getVoltSensorIndex(sensor_type);
  } catch (...) {
    return RSMI_STATUS_NOT_SUPPORTED;
  }
  CHK_API_SUPPORT_ONLY(voltage, metric, sensor_index)

  ret = get_dev_mon_value(mon_type, dv_ind, sensor_index, voltage);

  return ret;
  CATCH
}

rsmi_status_t
rsmi_dev_fan_speed_get(uint32_t dv_ind, uint32_t sensor_ind, int64_t *speed) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  rsmi_status_t ret;

  ++sensor_ind;  // fan sysfs files have 1-based indices

  CHK_SUPPORT_SUBVAR_ONLY(speed, sensor_ind)

  DEVICE_MUTEX

  ret = get_dev_mon_value(amd::smi::kMonFanSpeed, dv_ind, sensor_ind, speed);

  return ret;
  CATCH
}

rsmi_status_t
rsmi_dev_fan_rpms_get(uint32_t dv_ind, uint32_t sensor_ind, int64_t *speed) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  ++sensor_ind;  // fan sysfs files have 1-based indices

  CHK_SUPPORT_SUBVAR_ONLY(speed, sensor_ind)

  rsmi_status_t ret;

  DEVICE_MUTEX

  ret = get_dev_mon_value(amd::smi::kMonFanRPMs, dv_ind, sensor_ind, speed);

  return ret;
  CATCH
}

rsmi_status_t
rsmi_dev_fan_reset(uint32_t dv_ind, uint32_t sensor_ind) {
  TRY
  rsmi_status_t ret;
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  ++sensor_ind;  // fan sysfs files have 1-based indices
  REQUIRE_ROOT_ACCESS
  DEVICE_MUTEX
  ret = set_dev_mon_value<uint64_t>(amd::smi::kMonFanCntrlEnable,
                                                       dv_ind, sensor_ind, 2);
  return ret;

  CATCH
}

rsmi_status_t
rsmi_dev_fan_speed_set(uint32_t dv_ind, uint32_t sensor_ind, uint64_t speed) {
  TRY

  rsmi_status_t ret;
  uint64_t max_speed;
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  REQUIRE_ROOT_ACCESS
  DEVICE_MUTEX

  ret = rsmi_dev_fan_speed_max_get(dv_ind, sensor_ind, &max_speed);

  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  if (speed > max_speed) {
    return RSMI_STATUS_INPUT_OUT_OF_BOUNDS;
  }

  ++sensor_ind;  // fan sysfs files have 1-based indices

  // First need to set fan mode (pwm1_enable) to 1 (aka, "manual")
  ret = set_dev_mon_value<uint64_t>(amd::smi::kMonFanCntrlEnable, dv_ind,
                                                               sensor_ind, 1);
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  ret = set_dev_mon_value<uint64_t>(amd::smi::kMonFanSpeed, dv_ind,
                                                           sensor_ind, speed);
  return ret;

  CATCH
}

rsmi_status_t
rsmi_dev_fan_speed_max_get(uint32_t dv_ind, uint32_t sensor_ind,
                                                        uint64_t *max_speed) {
  TRY
  rsmi_status_t ret;
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  ++sensor_ind;  // fan sysfs files have 1-based indices
  CHK_SUPPORT_SUBVAR_ONLY(max_speed, sensor_ind)
  DEVICE_MUTEX

  ret = get_dev_mon_value(amd::smi::kMonMaxFanSpeed, dv_ind, sensor_ind,
                                      reinterpret_cast<int64_t *>(max_speed));

  return ret;
  CATCH
}

rsmi_status_t
rsmi_dev_od_volt_info_get(uint32_t dv_ind, rsmi_od_volt_freq_data_t *odv) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  DEVICE_MUTEX
  CHK_SUPPORT_NAME_ONLY(odv)
  rsmi_status_t ret = get_od_clk_volt_info(dv_ind, odv);

  return ret;
  CATCH
}

rsmi_status_t
rsmi_dev_gpu_reset(uint32_t dv_ind) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  REQUIRE_ROOT_ACCESS
  DEVICE_MUTEX

  rsmi_status_t ret;
  uint64_t status_code = 0;

  // Read amdgpu_gpu_recover to reset it
  ret = get_dev_value_int(amd::smi::kDevGpuReset, dv_ind, &status_code);
  return ret;

  CATCH
}

rsmi_status_t rsmi_dev_od_volt_curve_regions_get(uint32_t dv_ind,
                     uint32_t *num_regions, rsmi_freq_volt_region_t *buffer) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << " | ======= start =======";
  LOG_TRACE(ss);

  CHK_SUPPORT_NAME_ONLY((num_regions == nullptr || buffer == nullptr) ?
                                                        nullptr : num_regions)
  if (*num_regions == 0) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  DEVICE_MUTEX
  rsmi_status_t ret = get_od_clk_volt_curve_regions(dv_ind, num_regions,
                                                                      buffer);
  if (*num_regions == 0) {
    ret = RSMI_STATUS_NOT_SUPPORTED;
  }
  ss << __PRETTY_FUNCTION__ << " | ======= end ======= | returning "
     << getRSMIStatusString(ret);
  LOG_TRACE(ss);
  return ret;
  CATCH
}

rsmi_status_t
rsmi_dev_power_max_get(uint32_t dv_ind, uint32_t sensor_ind, uint64_t *power) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  (void)sensor_ind;  // Not used yet
  // ++sensor_ind;  // power sysfs files have 1-based indices
  CHK_SUPPORT_NAME_ONLY(power)

  rsmi_status_t ret;

  DEVICE_MUTEX
  ret = get_power_mon_value(amd::smi::kPowerMaxGPUPower, dv_ind, power);

  return ret;
  CATCH
}

rsmi_status_t
rsmi_dev_power_ave_get(uint32_t dv_ind, uint32_t sensor_ind, uint64_t *power) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  ++sensor_ind;  // power sysfs files have 1-based indices

  CHK_SUPPORT_SUBVAR_ONLY(power, sensor_ind)
  rsmi_status_t ret;

  DEVICE_MUTEX
  ret = get_dev_mon_value(amd::smi::kMonPowerAve, dv_ind, sensor_ind, power);

  return ret;
  CATCH
}

rsmi_status_t
rsmi_dev_current_socket_power_get(uint32_t dv_ind, uint64_t *socket_power) {
  TRY
  std::ostringstream ss;
  rsmi_status_t rsmiReturn = RSMI_STATUS_NOT_SUPPORTED;
  std::string val_str;
  uint32_t sensor_ind = 1;  // socket_power sysfs files have 1-based indices
  amd::smi::MonitorTypes mon_type = amd::smi::kMonPowerInput;
  ss << __PRETTY_FUNCTION__ << " | ======= start =======, dv_ind="
     << std::to_string(dv_ind);
  LOG_TRACE(ss);
  if (socket_power == nullptr) {
    rsmiReturn = RSMI_STATUS_INVALID_ARGS;
    ss << __PRETTY_FUNCTION__
       << " | ======= end ======= "
       << " | Fail "
       << " | Device #: " << dv_ind
       << " | Type: " << monitorTypesToString.at(mon_type)
       << " | Cause: socket_power was a null ptr reference"
       << " | Returning = "
       << getRSMIStatusString(rsmiReturn) << " |";
    LOG_ERROR(ss);
    return RSMI_STATUS_INVALID_ARGS;
  }
  CHK_SUPPORT_SUBVAR_ONLY(socket_power, sensor_ind)
  DEVICE_MUTEX

  if (dev->monitor() == nullptr) {
    ss << __PRETTY_FUNCTION__
       << " | ======= end ======= "
       << " | Fail "
       << " | Device #: " << dv_ind
       << " | Type: " << monitorTypesToString.at(mon_type)
       << " | Cause: hwmon monitor was a null ptr reference"
       << " | Returning = "
       << getRSMIStatusString(rsmiReturn) << " |";
    LOG_ERROR(ss);
    return rsmiReturn;
  }

  int ret = dev->monitor()->readMonitor(amd::smi::kMonPowerLabel,
                                        sensor_ind, &val_str);
  if (ret || val_str != "PPT" || val_str.size() != 3) {
    if (ret != 0) {
      rsmiReturn = amd::smi::ErrnoToRsmiStatus(ret);
    }
    ss << __PRETTY_FUNCTION__
       << " | ======= end ======= "
       << " | Fail "
       << " | Device #: " << dv_ind
       << " | Type: " << monitorTypesToString.at(mon_type)
       << " | Cause: readMonitor() returned an error status"
       << " or Socket Power label did not show PPT or size of label data was"
       << " unexpected"
       << " | Returning = "
       << getRSMIStatusString(rsmiReturn) << " |";
    LOG_ERROR(ss);
    return rsmiReturn;
  }
  rsmiReturn = get_dev_mon_value(mon_type, dv_ind, sensor_ind,
                                 socket_power);
  ss << __PRETTY_FUNCTION__
     << " | ======= end ======= "
     << " | Success "
     << " | Device #: " << dv_ind
     << " | Type: " << monitorTypesToString.at(mon_type)
     << " | Data: " << *socket_power
     << " | Returning = "
     << getRSMIStatusString(rsmiReturn) << " |";
  LOG_TRACE(ss);
  return rsmiReturn;
  CATCH
}

rsmi_status_t rsmi_dev_power_get(uint32_t dv_ind, uint64_t *power,
                                 RSMI_POWER_TYPE *type) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << " | ======= start =======, dv_ind="
     << std::to_string(dv_ind);
  LOG_TRACE(ss);
  rsmi_status_t ret = RSMI_STATUS_NOT_SUPPORTED;
  RSMI_POWER_TYPE temp_power_type = RSMI_INVALID_POWER;
  uint64_t temp_power = 0;

  if (type == nullptr || power == nullptr) {
    ret = RSMI_STATUS_INVALID_ARGS;
    ss << __PRETTY_FUNCTION__
       << " | ======= end ======= "
       << " | Fail "
       << " | Device #: " << dv_ind
       << " | Type: " << amd::smi::power_type_string(temp_power_type)
       << " | Cause: power or monitor type was a null ptr reference"
       << " | Returning = "
       << getRSMIStatusString(ret) << " |";
    LOG_ERROR(ss);
    return ret;
  }

  // only change return value on success, invalid otherwise
  rsmi_status_t temp_ret = rsmi_dev_current_socket_power_get(dv_ind, &temp_power);
  if (temp_ret == RSMI_STATUS_SUCCESS) {
    temp_power_type = RSMI_CURRENT_POWER;
    ret = temp_ret;
  } else {
    temp_ret = rsmi_dev_power_ave_get(dv_ind, 0, &temp_power);
    if (temp_ret == RSMI_STATUS_SUCCESS) {
      temp_power_type = RSMI_AVERAGE_POWER;
      ret = temp_ret;
    }
  }
  *power = temp_power;
  *type = temp_power_type;
  ss << __PRETTY_FUNCTION__
     << " | ======= end ======= "
     << " | Success "
     << " | Device #: " << dv_ind
     << " | Type: " << amd::smi::power_type_string(temp_power_type)
     << " | Data: " << *power
     << " | Returning = "
     << getRSMIStatusString(ret) << " |";
  LOG_TRACE(ss);
  return ret;
  CATCH
}

rsmi_status_t
rsmi_dev_energy_count_get(uint32_t dv_ind, uint64_t *power,
                          float *counter_resolution, uint64_t *timestamp) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  if (power == nullptr ||
      timestamp == nullptr) {
      return RSMI_STATUS_INVALID_ARGS;
  }

  rsmi_status_t ret;
  rsmi_gpu_metrics_t gpu_metrics;
  ret = rsmi_dev_gpu_metrics_info_get(dv_ind, &gpu_metrics);
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  *power = gpu_metrics.energy_accumulator;
  *timestamp = gpu_metrics.system_clock_counter;
  // hard-coded for now since all ASICs have same resolution. If it ASIC
  // dependent then this information should come from Kernel
  if (counter_resolution)
    *counter_resolution = kEnergyCounterResolution;

  return ret;
  CATCH
}

rsmi_status_t
rsmi_dev_power_cap_default_get(uint32_t dv_ind, uint64_t *default_cap) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  uint32_t sensor_ind = 1; // power sysfs files have 1-based indices
  CHK_SUPPORT_SUBVAR_ONLY(default_cap, sensor_ind)

  rsmi_status_t ret;

  DEVICE_MUTEX
  ret = get_dev_mon_value(amd::smi::kMonPowerCapDefault, dv_ind, sensor_ind, default_cap);

  return ret;
  CATCH
}

rsmi_status_t
rsmi_dev_power_cap_get(uint32_t dv_ind, uint32_t sensor_ind, uint64_t *cap) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  ++sensor_ind;  // power sysfs files have 1-based indices
  CHK_SUPPORT_SUBVAR_ONLY(cap, sensor_ind)

  rsmi_status_t ret;

  DEVICE_MUTEX
  ret = get_dev_mon_value(amd::smi::kMonPowerCap, dv_ind, sensor_ind, cap);

  return ret;
  CATCH
}

rsmi_status_t
rsmi_dev_power_cap_range_get(uint32_t dv_ind, uint32_t sensor_ind,
                                               uint64_t *max, uint64_t *min) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  ++sensor_ind;  // power sysfs files have 1-based indices
  CHK_SUPPORT_SUBVAR_ONLY((min == nullptr || max == nullptr ?nullptr : min),
                                                                   sensor_ind)
  rsmi_status_t ret;

  DEVICE_MUTEX
  ret = get_dev_mon_value(amd::smi::kMonPowerCapMax, dv_ind, sensor_ind, max);

  if (ret == RSMI_STATUS_SUCCESS) {
    ret = get_dev_mon_value(amd::smi::kMonPowerCapMin, dv_ind,
                                                             sensor_ind, min);
  }
  return ret;
  CATCH
}

rsmi_status_t
rsmi_dev_power_cap_set(uint32_t dv_ind, uint32_t sensor_ind, uint64_t cap) {
  TRY
  rsmi_status_t ret;
  uint64_t min;
  uint64_t max;
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  REQUIRE_ROOT_ACCESS
  DEVICE_MUTEX

  ret = rsmi_dev_power_cap_range_get(dv_ind, sensor_ind, &max, &min);
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  // All rsmi_* calls that use sensor_ind should use the 0-based value,
  // so increment this after the call above.
  ++sensor_ind;  // power sysfs files have 1-based indices

  if (cap > max || cap < min) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  ret = set_dev_mon_value<uint64_t>(amd::smi::kMonPowerCap, dv_ind,
                                                             sensor_ind, cap);

  return ret;
  CATCH
}

rsmi_status_t
rsmi_dev_power_profile_presets_get(uint32_t dv_ind, uint32_t reserved,
                                        rsmi_power_profile_status_t *status) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  (void)reserved;
  CHK_SUPPORT_NAME_ONLY(status)

  DEVICE_MUTEX
  rsmi_status_t ret = get_power_profiles(dv_ind, status, nullptr);

  return ret;
  CATCH
}

rsmi_status_t
rsmi_dev_power_profile_set(uint32_t dv_ind, uint32_t dummy,
                                  rsmi_power_profile_preset_masks_t profile) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  REQUIRE_ROOT_ACCESS

  (void)dummy;
  DEVICE_MUTEX
  rsmi_status_t ret = set_power_profile(dv_ind, profile);

  return ret;
  CATCH
}

rsmi_status_t
rsmi_dev_memory_total_get(uint32_t dv_ind, rsmi_memory_type_t mem_type,
                                                            uint64_t *total) {
  TRY
  rsmi_status_t ret;
  amd::smi::DevInfoTypes mem_type_file;
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  CHK_SUPPORT_VAR(total, mem_type)

  switch (mem_type) {
    case RSMI_MEM_TYPE_GTT:
      mem_type_file = amd::smi::kDevMemTotGTT;
      break;

    case RSMI_MEM_TYPE_VIS_VRAM:
      mem_type_file = amd::smi::kDevMemTotVisVRAM;
      break;

    case RSMI_MEM_TYPE_VRAM:
      mem_type_file = amd::smi::kDevMemTotVRAM;
      break;

    default:
      assert(false);  // Unexpected memory type
      return RSMI_STATUS_INVALID_ARGS;
  }

  DEVICE_MUTEX
  ret = get_dev_value_int(mem_type_file, dv_ind, total);

  // Fallback to KFD reported memory if VRAM total is 0
  if (mem_type == RSMI_MEM_TYPE_VRAM && *total == 0) {
    GET_DEV_AND_KFDNODE_FROM_INDX
    if (kfd_node->get_total_memory(total) == 0 && *total > 0) {
      ss << __PRETTY_FUNCTION__
         << " | inside success fallback... "
         << " | Device #: " << std::to_string(dv_ind)
         << " | Type = " << devInfoTypesStrings.at(mem_type_file)
         << " | Data: total = " << std::to_string(*total)
         << " | ret = " << getRSMIStatusString(RSMI_STATUS_SUCCESS);
      LOG_DEBUG(ss);
      return RSMI_STATUS_SUCCESS;
    }
  }

  ss << __PRETTY_FUNCTION__
     << " | after fallback... "
     << " | Device #: " << std::to_string(dv_ind)
     << " | Type = " << devInfoTypesStrings.at(mem_type_file)
     << " | Data: total = " << std::to_string(*total)
     << " | ret = " << getRSMIStatusString(ret);
  LOG_DEBUG(ss);
  return ret;
  CATCH
}
rsmi_status_t
rsmi_dev_memory_usage_get(uint32_t dv_ind, rsmi_memory_type_t mem_type,
                                                              uint64_t *used) {
  TRY
  rsmi_status_t ret;
  amd::smi::DevInfoTypes mem_type_file;
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  CHK_SUPPORT_VAR(used, mem_type)

  switch (mem_type) {
    case RSMI_MEM_TYPE_GTT:
      mem_type_file = amd::smi::kDevMemUsedGTT;
      break;

    case RSMI_MEM_TYPE_VIS_VRAM:
      mem_type_file = amd::smi::kDevMemUsedVisVRAM;
      break;

    case RSMI_MEM_TYPE_VRAM:
      mem_type_file = amd::smi::kDevMemUsedVRAM;
      break;

    default:
      assert(false);  // Unexpected memory type
      return RSMI_STATUS_INVALID_ARGS;
  }

  DEVICE_MUTEX
  ret = get_dev_value_int(mem_type_file, dv_ind, used);

  // Fallback to KFD reported memory if no VRAM
  if (mem_type == RSMI_MEM_TYPE_VRAM && *used == 0) {
    GET_DEV_AND_KFDNODE_FROM_INDX
    uint64_t total = 0;
    ret = get_dev_value_int(amd::smi::kDevMemTotVRAM, dv_ind, &total);
    if (total != 0) {
      ss << __PRETTY_FUNCTION__
         << " no fallback needed! - "
         << " | Device #: " << std::to_string(dv_ind)
         << " | Type = " << devInfoTypesStrings.at(mem_type_file)
         << " | Data: Used = " << std::to_string(*used)
         << " | Data: total = " << std::to_string(total)
         << " | ret = " << getRSMIStatusString(ret);
    LOG_DEBUG(ss);
      return ret;  // do not need to fallback
    }
    if ( kfd_node->get_used_memory(used) == 0 ) {
      ss << __PRETTY_FUNCTION__
         << " | in fallback == success ..."
         << " | Device #: " << std::to_string(dv_ind)
         << " | Type = " << devInfoTypesStrings.at(mem_type_file)
         << " | Data: Used = " << std::to_string(*used)
         << " | Data: total = " << std::to_string(total)
         << " | ret = " << getRSMIStatusString(RSMI_STATUS_SUCCESS);
      LOG_DEBUG(ss);
      return RSMI_STATUS_SUCCESS;
    }
  }
  ss << __PRETTY_FUNCTION__
     << " | at end!!!! after fallback ..."
     << " | Device #: " << std::to_string(dv_ind)
     << " | Type = " << devInfoTypesStrings.at(mem_type_file)
     << " | Data: Used = " << std::to_string(*used)
     << " | ret = " << getRSMIStatusString(ret);
  LOG_DEBUG(ss);

  return ret;
  CATCH
}

rsmi_status_t
rsmi_dev_memory_busy_percent_get(uint32_t dv_ind, uint32_t *busy_percent) {
  TRY
  rsmi_status_t ret;
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  CHK_SUPPORT_NAME_ONLY(busy_percent)

  uint64_t tmp_util = 0;

  DEVICE_MUTEX
  ret = get_dev_value_int(amd::smi::kDevMemBusyPercent, dv_ind, &tmp_util);

  if (tmp_util > 100) {
    return RSMI_STATUS_UNEXPECTED_DATA;
  }
  *busy_percent = static_cast<uint32_t>(tmp_util);
  return ret;
  CATCH
}

rsmi_status_t
rsmi_status_string(rsmi_status_t status, const char **status_string) {
  TRY
  if (status_string == nullptr) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  const size_t status_u = static_cast<size_t>(status);
  switch (status_u) {
    case RSMI_STATUS_SUCCESS:
      *status_string = "RSMI_STATUS_SUCCESS: The function has been executed"
                          " successfully.";
      break;

    case RSMI_STATUS_INVALID_ARGS:
      *status_string =
          "RSMI_STATUS_INVALID_ARGS: The provided arguments do not"
                " meet the preconditions required for calling this function.";
      break;

    case RSMI_STATUS_NOT_SUPPORTED:
      *status_string = "RSMI_STATUS_NOT_SUPPORTED: This function is not"
                                     " supported in the current environment.";
      break;

    case RSMI_STATUS_FILE_ERROR:
      *status_string =
        "RSMI_STATUS_FILE_ERROR: There was an error in finding or"
        " opening a file or directory. The operation may not be supported by "
        "this Linux kernel version.";
      break;

    case RSMI_STATUS_PERMISSION:
      *status_string = "RSMI_STATUS_PERMISSION: The user ID of the calling"
        " process does not have sufficient permission to execute a command."
        "  Often this is fixed by running as root (sudo).";
      break;

    case RSMI_STATUS_OUT_OF_RESOURCES:
      *status_string = "RSMI_STATUS_OUT_OF_RESOURCES: Unable to acquire memory "
                       "or other resource";
      break;

    case RSMI_STATUS_INTERNAL_EXCEPTION:
      *status_string = "RSMI_STATUS_INTERNAL_EXCEPTION: An internal exception "
                       "was caught";
      break;

    case RSMI_STATUS_INPUT_OUT_OF_BOUNDS:
      *status_string = "RSMI_STATUS_INPUT_OUT_OF_BOUNDS: The provided input is "
                       "out of allowable or safe range";
      break;

    case RSMI_STATUS_INIT_ERROR:
      *status_string = "RSMI_STATUS_INIT_ERROR: An error occurred during "
                       "initialization, during monitor discovery or when when "
                       "initializing internal data structures";
      break;

    case RSMI_STATUS_NOT_YET_IMPLEMENTED:
      *status_string = "RSMI_STATUS_NOT_YET_IMPLEMENTED: The called function "
                        "has not been implemented in this system for this "
                        "device type";
      break;

    case RSMI_STATUS_NOT_FOUND:
      *status_string = "RSMI_STATUS_NOT_FOUND: An item required to complete "
                       "the call was not found";
      break;

    case RSMI_STATUS_INSUFFICIENT_SIZE:
      *status_string = "RSMI_STATUS_INSUFFICIENT_SIZE: Not enough resources "
                       "were available to fully execute the call";
      break;

    case RSMI_STATUS_INTERRUPT:
      *status_string = "RSMI_STATUS_INTERRUPT: An interrupt occurred while "
                       "executing the function";
      break;

    case RSMI_STATUS_UNEXPECTED_SIZE:
      *status_string = "RSMI_STATUS_UNEXPECTED_SIZE: Data (usually from reading"
                       " a file) was out of range from what was expected";
      break;

    case RSMI_STATUS_NO_DATA:
      *status_string = "RSMI_STATUS_NO_DATA: No data was found (usually from "
                       "reading a file) where data was expected";
      break;

    case RSMI_STATUS_UNEXPECTED_DATA:
      *status_string = "RSMI_STATUS_UNEXPECTED_DATA: Data read (usually from "
                       "a file) or provided to function is "
                       "not what was expected";
      break;

    case RSMI_STATUS_BUSY:
      *status_string = "RSMI_STATUS_BUSY: A resource or mutex could not be "
                        "acquired because it is already being used";
    break;

    case RSMI_STATUS_REFCOUNT_OVERFLOW:
      *status_string = "RSMI_STATUS_REFCOUNT_OVERFLOW: An internal reference "
                       "counter exceeded INT32_MAX";
      break;

    case RSMI_STATUS_SETTING_UNAVAILABLE:
      *status_string = "RSMI_STATUS_SETTING_UNAVAILABLE: Requested setting is "
                        "unavailable for the current device";
      break;

    case RSMI_STATUS_AMDGPU_RESTART_ERR:
      *status_string = "RSMI_STATUS_AMDGPU_RESTART_ERR: Could not successfully "
                        "restart the amdgpu driver";
      break;

    case RSMI_STATUS_UNKNOWN_ERROR:
      *status_string = "RSMI_STATUS_UNKNOWN_ERROR: An unknown error prevented "
                       "the call from completing successfully";
      break;

    default:
      *status_string = "RSMI_STATUS_UNKNOWN_ERROR: An unknown error occurred";
      return RSMI_STATUS_UNKNOWN_ERROR;
  }
  return RSMI_STATUS_SUCCESS;
  CATCH
}

rsmi_status_t
rsmi_dev_busy_percent_get(uint32_t dv_ind, uint32_t *busy_percent) {
  TRY
  std::string val_str;
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  CHK_SUPPORT_NAME_ONLY(busy_percent)

  DEVICE_MUTEX
  rsmi_status_t ret = get_dev_value_str(amd::smi::kDevUsage, dv_ind,
                                                                    &val_str);
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  errno = 0;
  *busy_percent = static_cast<uint32_t>(strtoul(val_str.c_str(), nullptr, 10));

  if (*busy_percent > 100) {
    return RSMI_STATUS_UNEXPECTED_DATA;
  }
  assert(errno == 0);

  return RSMI_STATUS_SUCCESS;

  CATCH
}

rsmi_status_t
rsmi_utilization_count_get(uint32_t dv_ind,
                rsmi_utilization_counter_t utilization_counters[],
                uint32_t count,
                uint64_t *timestamp) {
  TRY

  if (timestamp == nullptr ||
      utilization_counters == nullptr) {
      return RSMI_STATUS_INVALID_ARGS;
  }

  rsmi_status_t ret;
  rsmi_gpu_metrics_t gpu_metrics;
  uint32_t val_ui32;

  ret = rsmi_dev_gpu_metrics_info_get(dv_ind, &gpu_metrics);
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  for (uint32_t index = 0 ; index < count; index++) {
    switch (utilization_counters[index].type) {
      case RSMI_COARSE_GRAIN_GFX_ACTIVITY:
        val_ui32 = gpu_metrics.gfx_activity_acc;
        break;
      case RSMI_COARSE_GRAIN_MEM_ACTIVITY:
        val_ui32 = gpu_metrics.mem_activity_acc;
        break;
      default:
        return RSMI_STATUS_INVALID_ARGS;
    }
    if (val_ui32 == UINT32_MAX) {
      return RSMI_STATUS_NOT_SUPPORTED;
    }
    utilization_counters[index].value = val_ui32;
  }

  *timestamp = gpu_metrics.system_clock_counter;

  return ret;
  CATCH
}

rsmi_status_t
rsmi_dev_activity_metric_get(uint32_t dv_ind,
                             rsmi_activity_metric_t activity_metric_type,
                             rsmi_activity_metric_counter_t* activity_metric_counter) {

  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  if (!activity_metric_counter) {
    ostrstream << __PRETTY_FUNCTION__
               << " | ======= end ======= "
               << " | Fail "
               << " | Device #: " << dv_ind
               << " | Metric Type: " << activity_metric_type
               << " | Cause: rsmi_activity_metric_counter_t was a null ptr reference"
               << " | Returning = "
               << getRSMIStatusString(RSMI_STATUS_INVALID_ARGS) << " |";
    LOG_ERROR(ostrstream);
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS);
  rsmi_gpu_metrics_t gpu_metrics;
  status_code = rsmi_dev_gpu_metrics_info_get(dv_ind, &gpu_metrics);
  if (status_code != rsmi_status_t::RSMI_STATUS_SUCCESS) {
    ostrstream << __PRETTY_FUNCTION__
               << " | ======= end ======= "
               << " | Fail "
               << " | Device #: " << dv_ind
               << " | Metric Type: " << activity_metric_type
               << " | Cause: rsmi_dev_gpu_metrics_info_get returned "
               << getRSMIStatusString(status_code)
               << " | Returning = "
               << status_code << " |";
    LOG_ERROR(ostrstream);
    return status_code;
  }

  if (activity_metric_type & rsmi_activity_metric_t::RSMI_ACTIVITY_GFX) {
    activity_metric_counter->average_gfx_activity = gpu_metrics.average_gfx_activity;
    ostrstream << __PRETTY_FUNCTION__
               << " | For GFX: " << activity_metric_counter->average_gfx_activity;
    LOG_INFO(ostrstream);
  }
  if (activity_metric_type & rsmi_activity_metric_t::RSMI_ACTIVITY_UMC) {
    activity_metric_counter->average_umc_activity = gpu_metrics.average_umc_activity;
    ostrstream << __PRETTY_FUNCTION__
               << " | For UMC: " << activity_metric_counter->average_umc_activity;
    LOG_INFO(ostrstream);
  }
  if (activity_metric_type & rsmi_activity_metric_t::RSMI_ACTIVITY_MM) {
    activity_metric_counter->average_mm_activity  = gpu_metrics.average_mm_activity;
    ostrstream << __PRETTY_FUNCTION__
               << " | For MM: " << activity_metric_counter->average_mm_activity;
    LOG_INFO(ostrstream);
  }

  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | Success "
             << " | Device #: " << dv_ind
             << " | Metric Type: " << activity_metric_type
             << " | Returning = "
             << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_activity_avg_mm_get(uint32_t dv_ind, uint16_t* avg_activity) {

  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  if (!avg_activity) {
    ostrstream << __PRETTY_FUNCTION__
               << " | ======= end ======= "
               << " | Fail "
               << " | Device #: " << dv_ind
               << " | Metric Type: " << rsmi_activity_metric_t::RSMI_ACTIVITY_MM
               << " | Cause: avg_activity was a null ptr reference"
               << " | Returning = "
               << getRSMIStatusString(RSMI_STATUS_INVALID_ARGS) << " |";
    LOG_ERROR(ostrstream);
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS);
  rsmi_activity_metric_counter_t activity_metric_counter;
  status_code = rsmi_dev_activity_metric_get(dv_ind, rsmi_activity_metric_t::RSMI_ACTIVITY_MM, &activity_metric_counter);
  avg_activity = &activity_metric_counter.average_mm_activity;

  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | Success "
             << " | Device #: " << dv_ind
             << " | Metric Type: " << rsmi_activity_metric_t::RSMI_ACTIVITY_MM
             << " | Returning = "
             << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_vbios_version_get(uint32_t dv_ind, char *vbios, uint32_t len) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  CHK_SUPPORT_NAME_ONLY(vbios)

  if (len == 0) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  std::string val_str;

  DEVICE_MUTEX
  int ret = dev->readDevInfo(amd::smi::kDevVBiosVer, &val_str);

  if (ret != 0) {
    return amd::smi::ErrnoToRsmiStatus(ret);
  }

  uint32_t ln = static_cast<uint32_t>(val_str.copy(vbios, len));

  vbios[std::min(len - 1, ln)] = '\0';

  if (len < (val_str.size() + 1)) {
    return RSMI_STATUS_INSUFFICIENT_SIZE;
  }
  return RSMI_STATUS_SUCCESS;

  CATCH
}

rsmi_status_t
rsmi_version_get(rsmi_version_t *version) {
  TRY

  if (version == nullptr) {
    return RSMI_STATUS_INVALID_ARGS;
  }
  version->major = rocm_smi_VERSION_MAJOR;
  version->minor = rocm_smi_VERSION_MINOR;
  version->patch = rocm_smi_VERSION_PATCH;
  version->build = rocm_smi_VERSION_BUILD;

  return RSMI_STATUS_SUCCESS;

  CATCH
}

static const char *kROCmDriverVersionPath = "/sys/module/amdgpu/version";

rsmi_status_t
rsmi_version_str_get(rsmi_sw_component_t component, char *ver_str,
                                                               uint32_t len) {
  if (ver_str == nullptr || len == 0) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  TRY

  int err;
  std::string val_str;
  std::string ver_path;

  switch (component) {
    case RSMI_SW_COMP_DRIVER:
      ver_path = kROCmDriverVersionPath;
      break;

    default:
      assert(false);  // Unexpected component type provided
      return RSMI_STATUS_INVALID_ARGS;
  }

  err = amd::smi::ReadSysfsStr(ver_path, &val_str);

  if (err != 0) {
    struct utsname buf;
    err = uname(&buf);

    if (err != 0) {
      return amd::smi::ErrnoToRsmiStatus(err);
    }

    val_str = buf.release;
  }

  uint32_t ln = static_cast<uint32_t>(val_str.copy(ver_str, len));

  ver_str[std::min(len - 1, ln)] = '\0';

  if (len < (val_str.size() + 1)) {
    return RSMI_STATUS_INSUFFICIENT_SIZE;
  }
  return RSMI_STATUS_SUCCESS;

  CATCH
}

rsmi_status_t rsmi_dev_serial_number_get(uint32_t dv_ind,
                                             char *serial_num, uint32_t len) {
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  CHK_SUPPORT_NAME_ONLY(serial_num)
  if (len == 0) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  TRY
  DEVICE_MUTEX

  std::string val_str;
  rsmi_status_t ret = get_dev_value_str(amd::smi::kDevSerialNumber,
                                                            dv_ind, &val_str);

  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  uint32_t ln = static_cast<uint32_t>(val_str.copy(serial_num, len));

  serial_num[std::min(len - 1, ln)] = '\0';

  if (len < (val_str.size() + 1)) {
    return RSMI_STATUS_INSUFFICIENT_SIZE;
  }
  return RSMI_STATUS_SUCCESS;

  CATCH
}

rsmi_status_t
rsmi_dev_pci_replay_counter_get(uint32_t dv_ind, uint64_t *counter) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  CHK_SUPPORT_NAME_ONLY(counter)

  rsmi_status_t ret;

  DEVICE_MUTEX
  ret = get_dev_value_int(amd::smi::kDevPCIEReplayCount, dv_ind, counter);
  return ret;

  CATCH
}

rsmi_status_t
rsmi_dev_unique_id_get(uint32_t dv_ind, uint64_t *unique_id) {
  TRY
  rsmi_status_t ret;
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  CHK_SUPPORT_NAME_ONLY(unique_id)

  DEVICE_MUTEX
  ret = get_dev_value_int(amd::smi::kDevUniqueId, dv_ind, unique_id);
  return ret;

  CATCH
}
rsmi_status_t
rsmi_dev_counter_create(uint32_t dv_ind, rsmi_event_type_t type,
                                           rsmi_event_handle_t *evnt_handle) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  REQUIRE_ROOT_ACCESS

  // Note we don't need to pass in the variant to CHK_SUPPORT_VAR because
  // the success of this call doesn't depend on a sysfs file existing.
  CHK_SUPPORT_NAME_ONLY(evnt_handle)
  DEVICE_MUTEX
  *evnt_handle = reinterpret_cast<uintptr_t>(
                                      new amd::smi::evt::Event(type, dv_ind));

  if (evnt_handle == nullptr) {
    return RSMI_STATUS_OUT_OF_RESOURCES;
  }

  return RSMI_STATUS_SUCCESS;
  CATCH
}

rsmi_status_t
rsmi_dev_counter_destroy(rsmi_event_handle_t evnt_handle) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  if (evnt_handle == 0) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  int ret = 0;
  amd::smi::evt::Event *evt =
                        reinterpret_cast<amd::smi::evt::Event *>(evnt_handle);
  uint32_t dv_ind = evt->dev_ind();
  DEVICE_MUTEX
  REQUIRE_ROOT_ACCESS

  ret = evt->stopCounter();

  delete evt;
  return amd::smi::ErrnoToRsmiStatus(ret);;
  CATCH
}

rsmi_status_t
rsmi_counter_control(rsmi_event_handle_t evt_handle,
                                 rsmi_counter_command_t cmd, void * /*unused*/) {
  TRY

  amd::smi::evt::Event *evt =
                         reinterpret_cast<amd::smi::evt::Event *>(evt_handle);
  amd::smi::pthread_wrap _pw(*amd::smi::GetMutex(evt->dev_ind()));
  amd::smi::ScopedPthread _lock(_pw);

  REQUIRE_ROOT_ACCESS

  int ret = 0;

  if (evt_handle == 0) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  switch (cmd) {
    case RSMI_CNTR_CMD_START:
      ret = evt->startCounter();
      break;

    case RSMI_CNTR_CMD_STOP:
      ret = evt->stopCounter();
      break;

    default:
      assert(false);  // Unexpected perf counter command
      return RSMI_STATUS_INVALID_ARGS;
  }
  return amd::smi::ErrnoToRsmiStatus(ret);

  CATCH
}

rsmi_status_t
rsmi_counter_read(rsmi_event_handle_t evt_handle,
                                                rsmi_counter_value_t *value) {
  TRY

  if (value == nullptr || evt_handle == 0) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  amd::smi::evt::Event *evt =
                         reinterpret_cast<amd::smi::evt::Event *>(evt_handle);

  uint32_t dv_ind = evt->dev_ind();
  DEVICE_MUTEX
  REQUIRE_ROOT_ACCESS

  uint32_t ret;

  ret = evt->getValue(value);

  // If value > 2^48, then an overflow has occurred. We need to discard this
  // value and re-read:
  if (ret == 0 && value->value > 0xFFFFFFFFFFFF) {
    ret = evt->getValue(value);
  }
  if (ret == 0) {
    return RSMI_STATUS_SUCCESS;
  }

  return RSMI_STATUS_UNEXPECTED_SIZE;
  CATCH
}

rsmi_status_t
rsmi_counter_available_counters_get(uint32_t dv_ind,
                                rsmi_event_group_t grp, uint32_t *available) {
  rsmi_status_t ret;

  TRY
  CHK_SUPPORT_VAR(available, grp)
  DEVICE_MUTEX
  uint64_t val;

  switch (grp) {
    case RSMI_EVNT_GRP_XGMI:
    case RSMI_EVNT_GRP_XGMI_DATA_OUT:

      ret = get_dev_value_int(amd::smi::kDevDFCountersAvailable, dv_ind, &val);
      assert(val < UINT32_MAX);
      *available = static_cast<uint32_t>(val);
      break;

    default:
      return RSMI_STATUS_INVALID_ARGS;
  }
  return ret;
  CATCH
}

rsmi_status_t
rsmi_dev_counter_group_supported(uint32_t dv_ind, rsmi_event_group_t group) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  DEVICE_MUTEX
  GET_DEV_FROM_INDX

  amd::smi::evt::dev_evt_grp_set_t *grp = dev->supported_event_groups();

  if (grp->find(group) == grp->end()) {
    return RSMI_STATUS_NOT_SUPPORTED;
  }

  return RSMI_STATUS_SUCCESS;
  CATCH
}

rsmi_status_t
rsmi_compute_process_info_get(rsmi_process_info_t *procs,
                                                        uint32_t *num_items) {
  TRY

  if (num_items == nullptr) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  uint32_t procs_found = 0;

  int err = amd::smi::GetProcessInfo(procs, *num_items, &procs_found);

  if (err) {
    return amd::smi::ErrnoToRsmiStatus(err);
  }

  if (procs && *num_items < procs_found) {
    return RSMI_STATUS_INSUFFICIENT_SIZE;
  }
  if (procs == nullptr || *num_items > procs_found) {
    *num_items = procs_found;
  }

  return RSMI_STATUS_SUCCESS;

  CATCH
}

rsmi_status_t
rsmi_compute_process_gpus_get(uint32_t pid, uint32_t *dv_indices,
                                                      uint32_t *num_devices) {
  TRY

  if (num_devices == nullptr) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  std::unordered_set<uint64_t> gpu_set;
  int err = amd::smi::GetProcessGPUs(pid, &gpu_set);

  if (err) {
    return amd::smi::ErrnoToRsmiStatus(err);
  }

  uint32_t i = 0;
  amd::smi::RocmSMI& smi = amd::smi::RocmSMI::getInstance();

  // filter out the devices not visible to container
  auto& nodes = smi.kfd_node_map();
  for (auto nit = gpu_set.begin(); nit != gpu_set.end();) {
    uint64_t gpu_id_val = (*nit);
    auto kfd_ite = nodes.find(gpu_id_val);
    if (kfd_ite == nodes.end()) {
      nit = gpu_set.erase(nit);
    } else {
      nit++;
    }
  }

  if (dv_indices != nullptr) {
    for (auto it = gpu_set.begin(); i < *num_devices && it != gpu_set.end();
                                                                  ++it, ++i) {
      uint64_t gpu_id_val = (*it);
      dv_indices[i] = nodes[gpu_id_val]->amdgpu_dev_index();
    }
  }

  if (dv_indices && *num_devices < gpu_set.size()) {
    // In this case, *num_devices should already hold the number of items
    // written to dv_devices. We just have to let the caller know there's more.
    return RSMI_STATUS_INSUFFICIENT_SIZE;
  }

  *num_devices = static_cast<uint32_t>(gpu_set.size());
  if (gpu_set.size() > smi.devices().size()) {
    return RSMI_STATUS_UNEXPECTED_SIZE;
  }

  return RSMI_STATUS_SUCCESS;

  CATCH
}

rsmi_status_t
rsmi_dev_memory_reserved_pages_get(uint32_t dv_ind, uint32_t *num_pages,
                                          rsmi_retired_page_record_t *records) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  rsmi_status_t ret;
  CHK_SUPPORT_NAME_ONLY(num_pages)

  std::vector<std::string> val_vec;

  ret = GetDevValueVec(amd::smi::kDevMemPageBad, dv_ind, &val_vec);

  // file is empty, which is valid for no errors
  if (ret == RSMI_STATUS_UNEXPECTED_DATA) {
    ret = RSMI_STATUS_SUCCESS;
  }
  if (ret == RSMI_STATUS_FILE_ERROR) {
    return RSMI_STATUS_NOT_SUPPORTED;
  }
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  if (records == nullptr || *num_pages > val_vec.size()) {
    *num_pages = static_cast<uint32_t>(val_vec.size());
  }
  if (records == nullptr) {
    return RSMI_STATUS_SUCCESS;
  }

  // Fill in records
  char status_code;
  rsmi_memory_page_status_t tmp_stat;
  std::string junk;

  for (uint32_t i = 0; i < *num_pages; ++i) {
    std::istringstream fs1(val_vec[i]);

    fs1 >> std::hex >> records[i].page_address;
    fs1 >> junk;
    assert(junk == ":");
    fs1 >> std::hex >> records[i].page_size;
    fs1 >> junk;
    assert(junk == ":");
    fs1 >> status_code;

    switch (status_code) {
      case 'P':
        tmp_stat = RSMI_MEM_PAGE_STATUS_PENDING;
        break;

      case 'F':
        tmp_stat = RSMI_MEM_PAGE_STATUS_UNRESERVABLE;
        break;

      case 'R':
        tmp_stat = RSMI_MEM_PAGE_STATUS_RESERVED;
        break;
      default:
        assert(false);  // Unexpected retired memory page status code read
        return RSMI_STATUS_UNKNOWN_ERROR;
    }
    records[i].status = tmp_stat;
  }
  if (*num_pages < val_vec.size()) {
    return RSMI_STATUS_INSUFFICIENT_SIZE;
  }

  return RSMI_STATUS_SUCCESS;

  CATCH
}

rsmi_status_t
rsmi_compute_process_info_by_pid_get(uint32_t pid,
                                                  rsmi_process_info_t *proc) {
  TRY

  if (proc == nullptr) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  std::unordered_set<uint64_t> gpu_set;
  amd::smi::RocmSMI& smi = amd::smi::RocmSMI::getInstance();
  auto it = smi.kfd_node_map().begin();

  while (it != smi.kfd_node_map().end()) {
    uint64_t gpu_id = it->first;
    gpu_set.insert(gpu_id);
    it++;
  }

  int err = amd::smi::GetProcessInfoForPID(pid, proc, &gpu_set);

  if (err) {
    return amd::smi::ErrnoToRsmiStatus(err);
  }

  return RSMI_STATUS_SUCCESS;

  CATCH
}

rsmi_status_t
rsmi_compute_process_info_by_device_get(uint32_t pid, uint32_t dv_ind,
                                          rsmi_process_info_t *proc) {
  TRY
  if (proc == nullptr) {
    return RSMI_STATUS_INVALID_ARGS;
  }
  // Check the device and kfdnode exist
  GET_DEV_AND_KFDNODE_FROM_INDX

  std::unordered_set<uint64_t> gpu_set;
  gpu_set.insert(dev->kfd_gpu_id());
  int err = amd::smi::GetProcessInfoForPID(pid, proc, &gpu_set);

  if (err) {
    return amd::smi::ErrnoToRsmiStatus(err);
  }

  return RSMI_STATUS_SUCCESS;

  CATCH
}

rsmi_status_t
rsmi_dev_xgmi_error_status(uint32_t dv_ind, rsmi_xgmi_status_t *status) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  CHK_SUPPORT_NAME_ONLY(status)

  rsmi_status_t ret;
  uint64_t status_code;

  DEVICE_MUTEX
  ret = get_dev_value_int(amd::smi::kDevXGMIError, dv_ind, &status_code);

  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  switch (status_code) {
    case 0:
      *status = RSMI_XGMI_STATUS_NO_ERRORS;
      break;

    case 1:
      *status = RSMI_XGMI_STATUS_ERROR;
      break;

    case 2:
      *status = RSMI_XGMI_STATUS_MULTIPLE_ERRORS;
      break;

    default:
      assert(false);  // Unexpected XGMI error status read
      return RSMI_STATUS_UNKNOWN_ERROR;
  }
  return RSMI_STATUS_SUCCESS;

  CATCH
}

rsmi_status_t
rsmi_dev_xgmi_error_reset(uint32_t dv_ind) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  DEVICE_MUTEX

  rsmi_status_t ret;
  uint64_t status_code;

  // Reading xgmi_error resets it
  ret = get_dev_value_int(amd::smi::kDevXGMIError, dv_ind, &status_code);
  return ret;

  CATCH
}

rsmi_status_t
rsmi_dev_xgmi_hive_id_get(uint32_t dv_ind, uint64_t *hive_id) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  if (hive_id == nullptr) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  GET_DEV_AND_KFDNODE_FROM_INDX

  *hive_id = kfd_node->xgmi_hive_id();

  return RSMI_STATUS_SUCCESS;
  CATCH
}

rsmi_status_t
rsmi_topo_get_numa_node_number(uint32_t dv_ind, uint32_t *numa_node) {
  TRY

  return topo_get_numa_node_number(dv_ind, numa_node);
  CATCH
}

rsmi_status_t
rsmi_topo_get_link_weight(uint32_t dv_ind_src, uint32_t dv_ind_dst,
                          uint64_t *weight) {
  TRY

  uint32_t dv_ind = dv_ind_src;
  GET_DEV_AND_KFDNODE_FROM_INDX
  DEVICE_MUTEX

  if (weight == nullptr) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  rsmi_status_t status;
  uint32_t node_ind_dst;
  int ret = smi.get_node_index(dv_ind_dst, &node_ind_dst);

  if (ret == 0) {
    amd::smi::IO_LINK_TYPE type;
    ret = kfd_node->get_io_link_type(node_ind_dst, &type);
    if (ret == 0) {
      if (type == amd::smi::IOLINK_TYPE_XGMI) {
        ret = kfd_node->get_io_link_weight(node_ind_dst, weight);
        if (ret == 0)
          status = RSMI_STATUS_SUCCESS;
        else
          status = RSMI_STATUS_INIT_ERROR;
      } else {
        assert(false);  // Unexpected IO Link type read
        status = RSMI_STATUS_NOT_SUPPORTED;
      }
    } else if (kfd_node->numa_node_type() == amd::smi::IOLINK_TYPE_PCIEXPRESS) {
      *weight = kfd_node->numa_node_weight();  // from src GPU to it's CPU node
      uint64_t numa_weight_dst = 0;
      status = topo_get_numa_node_weight(dv_ind_dst, &numa_weight_dst);
      // from dst GPU to it's CPU node
      if (status == RSMI_STATUS_SUCCESS) {
        *weight = *weight + numa_weight_dst;
        uint32_t numa_number_src = kfd_node->numa_node_number();
        uint32_t numa_number_dst;
        status = topo_get_numa_node_number(dv_ind_dst, &numa_number_dst);
        if (status == RSMI_STATUS_SUCCESS) {
          if (numa_number_src != numa_number_dst) {
            uint64_t io_link_weight;
            ret = smi.get_io_link_weight(numa_number_src, numa_number_dst,
                                         &io_link_weight);
            if (ret == 0) {
              *weight = *weight + io_link_weight;
              // from src numa CPU node to dst numa CPU node
            } else {
              *weight = *weight + 10;
              // More than one CPU hops, hard coded 10
            }
          }
          status = RSMI_STATUS_SUCCESS;
        } else {
          assert(false);  // Error to read numa node number
          status = RSMI_STATUS_INIT_ERROR;
        }
      } else {
        assert(false);  // Error to read numa node weight
        status = RSMI_STATUS_INIT_ERROR;
      }
    } else {
      status = RSMI_STATUS_NOT_SUPPORTED;
    }
  } else {
    status = RSMI_STATUS_INVALID_ARGS;
  }

  return status;
  CATCH
}

 rsmi_status_t
 rsmi_minmax_bandwidth_get(uint32_t dv_ind_src, uint32_t dv_ind_dst,
                           uint64_t *min_bandwidth, uint64_t *max_bandwidth) {
  TRY

  uint32_t dv_ind = dv_ind_src;
  GET_DEV_AND_KFDNODE_FROM_INDX
  DEVICE_MUTEX

  if (min_bandwidth == nullptr || max_bandwidth == nullptr) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  if (dv_ind_src == dv_ind_dst) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  rsmi_status_t status;
  uint32_t node_ind_dst;
  int ret = smi.get_node_index(dv_ind_dst, &node_ind_dst);

  if (ret != 0) {
    return RSMI_STATUS_INVALID_ARGS;
  }


  amd::smi::IO_LINK_TYPE type;
  ret = kfd_node->get_io_link_type(node_ind_dst, &type);
  if ( ret == 0 && type == amd::smi::IOLINK_TYPE_XGMI) {
      ret = kfd_node->get_io_link_bandwidth(node_ind_dst,max_bandwidth,
                                                               min_bandwidth);
      if (ret == 0)
        status = RSMI_STATUS_SUCCESS;
      else
        status = RSMI_STATUS_INIT_ERROR;
  } else {  // from src GPU to it's CPU node, or type not XGMI
    status = RSMI_STATUS_NOT_SUPPORTED;
  }

  return status;
  CATCH
}

rsmi_status_t
rsmi_topo_get_link_type(uint32_t dv_ind_src, uint32_t dv_ind_dst,
                        uint64_t *hops, RSMI_IO_LINK_TYPE *type) {
  TRY

  uint32_t dv_ind = dv_ind_src;
  GET_DEV_AND_KFDNODE_FROM_INDX

  if (hops == nullptr) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  if (type == nullptr) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  rsmi_status_t status;
  uint32_t node_ind_dst;

  // handle the link type for CPU
  if (dv_ind_dst == CPU_NODE_INDEX) {
    // No CPU connected
    if (kfd_node->numa_node_weight() == 0) {
      return RSMI_STATUS_NOT_SUPPORTED;
    }
    amd::smi::IO_LINK_TYPE io_link_type =
              kfd_node->numa_node_type();
    switch (io_link_type) {
      case amd::smi::IOLINK_TYPE_XGMI:
        *type = RSMI_IOLINK_TYPE_XGMI;
        *hops = 1;
        return RSMI_STATUS_SUCCESS;
      case amd::smi::IOLINK_TYPE_PCIEXPRESS:
        *type = RSMI_IOLINK_TYPE_PCIEXPRESS;
        // always be the same CPU node
        *hops = 2;
        return RSMI_STATUS_SUCCESS;
      default:
        return RSMI_STATUS_NOT_SUPPORTED;
    }
  }

  int ret = smi.get_node_index(dv_ind_dst, &node_ind_dst);

  if (ret == 0) {
    amd::smi::IO_LINK_TYPE io_link_type;
    ret = kfd_node->get_io_link_type(node_ind_dst, &io_link_type);
    if (ret == 0) {
      if (io_link_type == amd::smi::IOLINK_TYPE_XGMI) {
        *type = RSMI_IOLINK_TYPE_XGMI;
        *hops = 1;
        status = RSMI_STATUS_SUCCESS;
      } else {
        assert(false);  // Unexpected IO Link type read
        status = RSMI_STATUS_NOT_SUPPORTED;
      }
    } else if (kfd_node->numa_node_type() == amd::smi::IOLINK_TYPE_PCIEXPRESS) {
      uint32_t numa_number_dst;
      status = topo_get_numa_node_number(dv_ind_dst, &numa_number_dst);
      if (status == RSMI_STATUS_SUCCESS) {
        uint32_t numa_number_src = kfd_node->numa_node_number();
        if (numa_number_src == numa_number_dst) {
          *hops = 2;  // same CPU node
        } else {
          uint64_t io_link_weight;
          ret = smi.get_io_link_weight(numa_number_src, numa_number_dst,
                                       &io_link_weight);
          if (ret == 0)
            *hops = 3;  // from src CPU node to dst CPU node
          else
            *hops = 4;  // More than one CPU hops, hard coded as 4
        }
        *type = RSMI_IOLINK_TYPE_PCIEXPRESS;
        status = RSMI_STATUS_SUCCESS;
      } else {
        assert(false);  // Error to get numa node number
        status = RSMI_STATUS_INIT_ERROR;
      }
    } else {
      status = RSMI_STATUS_NOT_SUPPORTED;
    }
  } else {
    status = RSMI_STATUS_INVALID_ARGS;
  }

  return status;
  CATCH
}

rsmi_status_t
rsmi_is_P2P_accessible(uint32_t dv_ind_src, uint32_t dv_ind_dst,
                       bool *accessible) {
  TRY

  uint32_t dv_ind = dv_ind_src;
  GET_DEV_AND_KFDNODE_FROM_INDX

  if (accessible == nullptr) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  uint32_t node_ind_src;
  uint32_t node_ind_dst;
  // Fetch the source and destination GPU node index
  if (smi.get_node_index(dv_ind_src, &node_ind_src) ||
      smi.get_node_index(dv_ind_dst, &node_ind_dst)) {
    *accessible = false;
    return RSMI_STATUS_INVALID_ARGS;
  }
  // If source device is same as destination, return true
  if (dv_ind_src == dv_ind_dst) {
    *accessible = true;
    return RSMI_STATUS_SUCCESS;
  }
  std::map<uint32_t, std::shared_ptr<amd::smi::IOLink>> io_link_map_tmp;
  std::map<uint32_t, std::shared_ptr<amd::smi::IOLink>>::iterator it;
  // Iterate over P2P links
  if (DiscoverP2PLinksPerNode(node_ind_src, &io_link_map_tmp) == 0) {
    for (it = io_link_map_tmp.begin(); it != io_link_map_tmp.end(); it++) {
      if(it->first == node_ind_dst) {
        *accessible = true;
        return RSMI_STATUS_SUCCESS;
      }
    }
    io_link_map_tmp.clear();
  } else {
    *accessible = false;
    return RSMI_STATUS_FILE_ERROR;
  }
  // Iterate over IO links
  if (DiscoverIOLinksPerNode(node_ind_src, &io_link_map_tmp) == 0) {
    for (it = io_link_map_tmp.begin(); it != io_link_map_tmp.end(); it++) {
      if(it->first == node_ind_dst) {
        *accessible = true;
        return RSMI_STATUS_SUCCESS;
      }
    }
  } else {
    *accessible = false;
    return RSMI_STATUS_FILE_ERROR;
  }
  *accessible = false;
  return RSMI_STATUS_SUCCESS;
  CATCH
}

static rsmi_status_t
get_compute_partition(uint32_t dv_ind, std::string &compute_partition) {
  TRY
  CHK_SUPPORT_NAME_ONLY(compute_partition.c_str())
  std::string compute_partition_str;

  DEVICE_MUTEX
  rsmi_status_t ret = get_dev_value_str(amd::smi::kDevComputePartition,
                                        dv_ind, &compute_partition_str);
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  switch (mapStringToRSMIComputePartitionTypes.at(compute_partition_str)) {
    case RSMI_COMPUTE_PARTITION_CPX:
    case RSMI_COMPUTE_PARTITION_SPX:
    case RSMI_COMPUTE_PARTITION_DPX:
    case RSMI_COMPUTE_PARTITION_TPX:
    case RSMI_COMPUTE_PARTITION_QPX:
      break;
    case RSMI_COMPUTE_PARTITION_INVALID:
    default:
      // Retrieved an unknown compute partition
      return RSMI_STATUS_UNEXPECTED_DATA;
  }
  compute_partition = compute_partition_str;
  return RSMI_STATUS_SUCCESS;
  CATCH
}

rsmi_status_t
rsmi_dev_compute_partition_get(uint32_t dv_ind, char *compute_partition,
                               uint32_t len) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======, dv_ind = "
     << dv_ind;
  LOG_TRACE(ss);
  if ((len == 0) || (compute_partition == nullptr)) {
    ss << __PRETTY_FUNCTION__
       << " | ======= end ======= "
       << " | Fail "
       << " | Device #: " << dv_ind
       << " | Type: "
       << devInfoTypesStrings.at(amd::smi::kDevComputePartition)
       << " | Cause: len was 0 or compute_partition variable was null"
       << " | Returning = "
       << getRSMIStatusString(RSMI_STATUS_INVALID_ARGS) << " |";
    LOG_ERROR(ss);
    return RSMI_STATUS_INVALID_ARGS;
  }
  CHK_SUPPORT_NAME_ONLY(compute_partition)

  std::string returning_compute_partition;
  rsmi_status_t ret = get_compute_partition(dv_ind,
                               returning_compute_partition);

  if (ret != RSMI_STATUS_SUCCESS) {
    ss << __PRETTY_FUNCTION__
       << " | ======= end ======= "
       << " | Fail "
       << " | Device #: " << dv_ind
       << " | Type: "
       << devInfoTypesStrings.at(amd::smi::kDevComputePartition)
       << " | Cause: could not retrieve current compute partition"
       << " | Returning = "
       << getRSMIStatusString(ret) << " |";
    LOG_ERROR(ss);
    return ret;
  }

  std::size_t length = returning_compute_partition.copy(compute_partition, len);
  compute_partition[length]='\0';

  if (len < (returning_compute_partition.size() + 1)) {
    ss << __PRETTY_FUNCTION__
       << " | ======= end ======= "
       << " | Fail "
       << " | Device #: " << dv_ind
       << " | Type: "
       << devInfoTypesStrings.at(amd::smi::kDevComputePartition)
       << " | Cause: requested size was insufficient"
       << " | Returning = "
       << getRSMIStatusString(RSMI_STATUS_INSUFFICIENT_SIZE) << " |";
    LOG_ERROR(ss);
    return RSMI_STATUS_INSUFFICIENT_SIZE;
  }
  ss << __PRETTY_FUNCTION__
     << " | ======= end ======= "
     << " | Success "
     << " | Device #: " << dv_ind
     << " | Type: "
     << devInfoTypesStrings.at(amd::smi::kDevComputePartition)
     << " | Data: " << compute_partition
     << " | Returning = "
     << getRSMIStatusString(ret) << " |";
  LOG_TRACE(ss);
  return ret;
  CATCH
}

static rsmi_status_t
is_available_compute_partition(uint32_t dv_ind,
                               std::string new_compute_partition) {
  TRY
  DEVICE_MUTEX
  std::string availableComputePartitions;
  rsmi_status_t ret =
      get_dev_value_line(amd::smi::kDevAvailableComputePartition,
                         dv_ind, &availableComputePartitions);
  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  bool isComputePartitionAvailable =
      amd::smi::containsString(availableComputePartitions,
                               new_compute_partition);
  return (isComputePartitionAvailable) ? RSMI_STATUS_SUCCESS :
                                         RSMI_STATUS_SETTING_UNAVAILABLE;
  CATCH
}

rsmi_status_t
rsmi_dev_compute_partition_set(uint32_t dv_ind,
                              rsmi_compute_partition_type_t compute_partition) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  REQUIRE_ROOT_ACCESS
  if (!amd::smi::is_sudo_user()) {
    return RSMI_STATUS_PERMISSION;
  }
  DEVICE_MUTEX
  std::string newComputePartitionStr
              = mapRSMIToStringComputePartitionTypes.at(compute_partition);
  std::string currentComputePartition;

  switch (compute_partition) {
    case RSMI_COMPUTE_PARTITION_CPX:
    case RSMI_COMPUTE_PARTITION_SPX:
    case RSMI_COMPUTE_PARTITION_DPX:
    case RSMI_COMPUTE_PARTITION_TPX:
    case RSMI_COMPUTE_PARTITION_QPX:
      break;
    case RSMI_COMPUTE_PARTITION_INVALID:
    default:
      ss << __PRETTY_FUNCTION__
         << " | ======= end ======= "
         << " | Fail "
         << " | Device #: " << dv_ind
         << " | Type: "
         << devInfoTypesStrings.at(amd::smi::kDevComputePartition)
         << " | Data: " << newComputePartitionStr
         << " | Cause: requested setting was invalid"
         << " | Returning = "
         << getRSMIStatusString(RSMI_STATUS_INVALID_ARGS) << " |";
      LOG_ERROR(ss);
      return RSMI_STATUS_INVALID_ARGS;
  }

  // Confirm what we are trying to set is available, otherwise provide
  // RSMI_STATUS_SETTING_UNAVAILABLE
  rsmi_status_t available_ret =
      is_available_compute_partition(dv_ind, newComputePartitionStr);
  if (available_ret != RSMI_STATUS_SUCCESS) {
    ss << __PRETTY_FUNCTION__
       << " | ======= end ======= "
       << " | Fail "
       << " | Device #: " << dv_ind
       << " | Type: "
       << devInfoTypesStrings.at(amd::smi::kDevComputePartition)
       << " | Data: " << newComputePartitionStr
       << " | Cause: not an available compute partition setting"
       << " | Returning = "
       << getRSMIStatusString(available_ret) << " |";
    LOG_ERROR(ss);
    return available_ret;
  }

  // do nothing if compute_partition is the current compute partition
  rsmi_status_t ret_get =
                        get_compute_partition(dv_ind, currentComputePartition);
  // we can try to set, even if we get unexpected data
  if (ret_get != RSMI_STATUS_SUCCESS
      && ret_get != RSMI_STATUS_UNEXPECTED_DATA) {
    ss << __PRETTY_FUNCTION__
       << " | ======= end ======= "
       << " | Fail "
       << " | Device #: " << dv_ind
       << " | Type: "
       << devInfoTypesStrings.at(amd::smi::kDevComputePartition)
       << " | Cause: could retrieve current compute partition or retrieved"
       << " unexpected data"
       << " | Returning = "
       << getRSMIStatusString(ret_get) << " |";
    LOG_ERROR(ss);
    return ret_get;
  }
  rsmi_compute_partition_type_t currRSMIComputePartition
    = mapStringToRSMIComputePartitionTypes.at(currentComputePartition);
  if (currRSMIComputePartition == compute_partition) {
    ss << __PRETTY_FUNCTION__
       << " | ======= end ======= "
       << " | Success - compute partition was already set at requested value"
       << " | Device #: " << dv_ind
       << " | Type: "
       << devInfoTypesStrings.at(amd::smi::kDevComputePartition)
       << " | Data: " << newComputePartitionStr
       << " | Returning = "
       << getRSMIStatusString(RSMI_STATUS_SUCCESS) << " |";
    LOG_TRACE(ss);
    return RSMI_STATUS_SUCCESS;
  }

  ss <<  __PRETTY_FUNCTION__ << " | about to try writing |"
     << newComputePartitionStr
     << "| size of string = " << newComputePartitionStr.size()
     << "| size of c-string = "<< std::dec
     << sizeof(newComputePartitionStr.c_str())/sizeof(newComputePartitionStr[0])
     << "| sizeof string = " << std::dec
     << sizeof(newComputePartitionStr);
  LOG_DEBUG(ss);

  GET_DEV_FROM_INDX
  int ret = dev->writeDevInfo(amd::smi::kDevComputePartition,
                              newComputePartitionStr);
  rsmi_status_t returnResponse = amd::smi::ErrnoToRsmiStatus(ret);
  ss << __PRETTY_FUNCTION__
     << " | ======= end ======= "
     << " | Success "
     << " | Device #: " << dv_ind
     << " | Type: "
     << devInfoTypesStrings.at(amd::smi::kDevComputePartition)
     << " | Data: " << newComputePartitionStr
     << " | Returning = "
     << getRSMIStatusString(returnResponse) << " |";
  LOG_TRACE(ss);

  // TODO(charpoag): investigate providing GPU busy state occured with
  return returnResponse;
  CATCH
}

static rsmi_status_t get_memory_partition(uint32_t dv_ind,
                                          std::string &memory_partition) {
  TRY
  CHK_SUPPORT_NAME_ONLY(memory_partition.c_str())
  std::string val_str;

  DEVICE_MUTEX
  rsmi_status_t ret = get_dev_value_str(amd::smi::kDevMemoryPartition,
                                        dv_ind, &val_str);

  if (ret != RSMI_STATUS_SUCCESS) {
    return ret;
  }

  switch (mapStringToMemoryPartitionTypes.at(val_str)) {
    case RSMI_MEMORY_PARTITION_NPS1:
    case RSMI_MEMORY_PARTITION_NPS2:
    case RSMI_MEMORY_PARTITION_NPS4:
    case RSMI_MEMORY_PARTITION_NPS8:
      break;
    case RSMI_MEMORY_PARTITION_UNKNOWN:
    default:
      // Retrieved an unknown memory partition
      return RSMI_STATUS_UNEXPECTED_DATA;
  }
  memory_partition = val_str;
  return RSMI_STATUS_SUCCESS;
  CATCH
}

rsmi_status_t
rsmi_dev_memory_partition_set(uint32_t dv_ind,
                              rsmi_memory_partition_type_t memory_partition) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  REQUIRE_ROOT_ACCESS
  DEVICE_MUTEX
  bool isCorrectDevice = false;
  char boardName[128];
  boardName[0] = '\0';
  // rsmi_dev_memory_partition_set is only available for for discrete variant,
  // others are required to update through bios settings
  rsmi_dev_name_get(dv_ind, boardName, 128);
  std::string myBoardName = boardName;
  if (!myBoardName.empty()) {
    std::transform(myBoardName.begin(), myBoardName.end(), myBoardName.begin(),
                   ::tolower);
    if (myBoardName.find("mi") != std::string::npos &&
        myBoardName.find("00x") != std::string::npos) {
      isCorrectDevice = true;
    }
  }

  if (!isCorrectDevice) {
    ss << __PRETTY_FUNCTION__
         << " | ======= end ======= "
         << " | Fail "
         << " | Device #: " << dv_ind
         << " | Type: "
         << devInfoTypesStrings.at(amd::smi::kDevMemoryPartition)
         << " | Cause: device board name does not support this action"
         << " | Returning = "
         << getRSMIStatusString(RSMI_STATUS_NOT_SUPPORTED) << " |";
      LOG_ERROR(ss);
    return RSMI_STATUS_NOT_SUPPORTED;
  }

  switch (memory_partition) {
    case RSMI_MEMORY_PARTITION_NPS1:
    case RSMI_MEMORY_PARTITION_NPS2:
    case RSMI_MEMORY_PARTITION_NPS4:
    case RSMI_MEMORY_PARTITION_NPS8:
      break;
    case RSMI_MEMORY_PARTITION_UNKNOWN:
    default:
      ss << __PRETTY_FUNCTION__
         << " | ======= end ======= "
         << " | Fail "
         << " | Device #: " << dv_ind
         << " | Type: "
         << devInfoTypesStrings.at(amd::smi::kDevMemoryPartition)
         << " | Cause: requested setting was invalid"
         << " | Returning = "
         << getRSMIStatusString(RSMI_STATUS_INVALID_ARGS) << " |";
      LOG_ERROR(ss);
      return RSMI_STATUS_INVALID_ARGS;
  }
  std::string newMemoryPartition
              = mapRSMIToStringMemoryPartitionTypes.at(memory_partition);
  std::string currentMemoryPartition;

  // do nothing if memory_partition is the current mode
  rsmi_status_t ret_get = get_memory_partition(dv_ind, currentMemoryPartition);
  // we can try to set, even if we get unexpected data
  if (ret_get != RSMI_STATUS_SUCCESS
      && ret_get != RSMI_STATUS_UNEXPECTED_DATA) {
    ss << __PRETTY_FUNCTION__
       << " | ======= end ======= "
       << " | Fail "
       << " | Device #: " << dv_ind
       << " | Type: "
       << devInfoTypesStrings.at(amd::smi::kDevMemoryPartition)
       << " | Cause: could retrieve current memory partition or retrieved"
       << " unexpected data"
       << " | Returning = "
       << getRSMIStatusString(ret_get) << " |";
    LOG_ERROR(ss);
    return ret_get;
  }
  rsmi_memory_partition_type_t currRSMIMemoryPartition
    = mapStringToMemoryPartitionTypes.at(currentMemoryPartition);
  if (currRSMIMemoryPartition == memory_partition) {
    ss << __PRETTY_FUNCTION__
     << " | ======= end ======= "
     << " | Success - no change, current memory partition was already requested"
     << " setting"
     << " | Device #: " << dv_ind
     << " | Type: "
     << devInfoTypesStrings.at(amd::smi::kDevMemoryPartition)
     << " | Data: " << newMemoryPartition
     << " | Returning = "
     << getRSMIStatusString(RSMI_STATUS_SUCCESS) << " |";
    LOG_TRACE(ss);
    return RSMI_STATUS_SUCCESS;
  }

  GET_DEV_FROM_INDX
  int ret = dev->writeDevInfo(amd::smi::kDevMemoryPartition,
                              newMemoryPartition);

  if (amd::smi::ErrnoToRsmiStatus(ret) != RSMI_STATUS_SUCCESS) {
    rsmi_status_t err = amd::smi::ErrnoToRsmiStatus(ret);
    if (ret == EACCES) {
      err = RSMI_STATUS_NOT_SUPPORTED;  // already verified permissions
    }
    ss << __PRETTY_FUNCTION__
       << " | ======= end ======= "
       << " | Fail "
       << " | Device #: " << dv_ind
       << " | Type: "
       << devInfoTypesStrings.at(amd::smi::kDevMemoryPartition)
       << " | Cause: issue writing reqested setting of " + newMemoryPartition
       << " | Returning = "
       << getRSMIStatusString(err) << " |";
    LOG_ERROR(ss);
    return err;
  }

  rsmi_status_t restartRet = dev->restartAMDGpuDriver();
  ss << __PRETTY_FUNCTION__
     << " | ======= end ======= "
     << " | Success - if restart completed successfully"
     << " | Device #: " << dv_ind
     << " | Type: "
     << devInfoTypesStrings.at(amd::smi::kDevMemoryPartition)
     << " | Data: " << newMemoryPartition
     << " | Returning = "
     << getRSMIStatusString(restartRet) << " |";
  LOG_TRACE(ss);
  return restartRet;
  CATCH
}

rsmi_status_t
rsmi_dev_memory_partition_get(uint32_t dv_ind, char *memory_partition,
                               uint32_t len) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  if ((len == 0) || (memory_partition == nullptr)) {
    ss << __PRETTY_FUNCTION__
       << " | ======= end ======= "
       << " | Fail "
       << " | Device #: " << dv_ind
       << " | Type: "
       << devInfoTypesStrings.at(amd::smi::kDevMemoryPartition)
       << " | Cause: user sent invalid arguments, len = 0 or memory partition"
       << " was a null ptr"
       << " | Returning = "
       << getRSMIStatusString(RSMI_STATUS_INVALID_ARGS) << " |";
    LOG_ERROR(ss);
    return RSMI_STATUS_INVALID_ARGS;
  }
  CHK_SUPPORT_NAME_ONLY(memory_partition)

  std::string returning_memory_partition;
  rsmi_status_t ret = get_memory_partition(dv_ind,
                               returning_memory_partition);

  if (ret != RSMI_STATUS_SUCCESS) {
    ss << __PRETTY_FUNCTION__
       << " | ======= end ======= "
       << " | Fail "
       << " | Device #: " << dv_ind
       << " | Type: "
       << devInfoTypesStrings.at(amd::smi::kDevMemoryPartition)
       << " | Cause: could not successfully retrieve current memory partition "
       << " | Returning = "
       << getRSMIStatusString(ret) << " |";
    LOG_ERROR(ss);
    return ret;
  }

  std::size_t buff_size =
      returning_memory_partition.copy(memory_partition, len);
  memory_partition[buff_size] = '\0';

  if (len < (returning_memory_partition.size() + 1)) {
    ss << __PRETTY_FUNCTION__
       << " | ======= end ======= "
       << " | Fail "
       << " | Device #: " << dv_ind
       << " | Type: "
       << devInfoTypesStrings.at(amd::smi::kDevMemoryPartition)
       << " | Cause: could not successfully retrieve current memory partition "
       << " | Returning = "
       << getRSMIStatusString(ret) << " |";
    LOG_ERROR(ss);
    return RSMI_STATUS_INSUFFICIENT_SIZE;
  }
  ss << __PRETTY_FUNCTION__
     << " | ======= end ======= "
     << " | Success "
     << " | Device #: " << dv_ind
     << " | Type: "
     << devInfoTypesStrings.at(amd::smi::kDevMemoryPartition)
     << " | Data: " << memory_partition
     << " | Returning = "
     << getRSMIStatusString(ret) << " |";
  LOG_TRACE(ss);
  return ret;
  CATCH
}

rsmi_status_t rsmi_dev_compute_partition_reset(uint32_t dv_ind) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  REQUIRE_ROOT_ACCESS
  DEVICE_MUTEX
  GET_DEV_FROM_INDX
  rsmi_status_t ret = RSMI_STATUS_NOT_SUPPORTED;

  // Only use 1st index, rest are there in-case of future issues
  // NOTE: Partitions sets cause rocm-smi indexes to fluctuate
  // since the nodes are grouped in respect to primary node - why we only use
  // 1st node/device id to reset
  std::string bootState =
          dev->readBootPartitionState<rsmi_compute_partition_type_t>(0);

  // Initiate reset
  // If bootState is UNKNOWN, we cannot reset - return RSMI_STATUS_NOT_SUPPORTED
  // Likely due to device not supporting it
  if (bootState != "UNKNOWN") {
    rsmi_compute_partition_type_t compute_partition =
      mapStringToRSMIComputePartitionTypes.at(bootState);
    ret = rsmi_dev_compute_partition_set(dv_ind, compute_partition);
  }
  ss << __PRETTY_FUNCTION__
     << " | ======= end ======= "
     << " | Success - if original boot state was not unknown or valid setting"
     << " | Device #: " << dv_ind
     << " | Type: "
     << devInfoTypesStrings.at(amd::smi::kDevComputePartition)
     << " | Data: " << bootState
     << " | Returning = "
     << getRSMIStatusString(ret) << " |";
  LOG_TRACE(ss);
  return ret;
  CATCH
}

rsmi_status_t rsmi_dev_memory_partition_reset(uint32_t dv_ind) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  REQUIRE_ROOT_ACCESS
  DEVICE_MUTEX
  GET_DEV_FROM_INDX
  rsmi_status_t ret = RSMI_STATUS_NOT_SUPPORTED;

  // Only use 1st index, rest are there in-case of future issues
  // NOTE: Partitions sets cause rocm-smi indexes to fluctuate.
  // Since the nodes are grouped in respect to primary node - why we only use
  // 1st node/device id to reset
  std::string bootState =
          dev->readBootPartitionState<rsmi_memory_partition_type_t>(0);

  // Initiate reset
  // If bootState is UNKNOWN, we cannot reset - return RSMI_STATUS_NOT_SUPPORTED
  // Likely due to device not supporting it
  if (bootState != "UNKNOWN") {
    rsmi_memory_partition_type_t memory_partition =
      mapStringToMemoryPartitionTypes.at(bootState);
    ret = rsmi_dev_memory_partition_set(dv_ind, memory_partition);
  }
  ss << __PRETTY_FUNCTION__
     << " | ======= end ======= "
     << " | Success - if original boot state was not unknown or valid setting"
     << " | Device #: " << dv_ind
     << " | Type: "
     << devInfoTypesStrings.at(amd::smi::kDevMemoryPartition)
     << " | Data: " << bootState
     << " | Returning = "
     << getRSMIStatusString(ret) << " |";
  LOG_TRACE(ss);
  return ret;
  CATCH
}

rsmi_status_t rsmi_dev_target_graphics_version_get(uint32_t dv_ind,
                                            uint64_t *gfx_version) {
    TRY
    std::ostringstream ss;
    ss << __PRETTY_FUNCTION__ << "| ======= start =======";
    rsmi_status_t ret = RSMI_STATUS_NOT_SUPPORTED;
    std::string version = "";
    const uint64_t undefined_gfx_version = std::numeric_limits<uint64_t>::max();
    LOG_TRACE(ss);
    if (gfx_version == nullptr) {
      ret = RSMI_STATUS_INVALID_ARGS;
    } else {
      *gfx_version = undefined_gfx_version;
      ret = amd::smi::rsmi_get_gfx_target_version(dv_ind , &version);
    }
    if (ret == RSMI_STATUS_SUCCESS) {
      version = amd::smi::removeString(version, "gfx");
      *gfx_version = std::stoull(version);
    }
    ss << __PRETTY_FUNCTION__
       << " | ======= end ======= "
       << " | Returning: " << getRSMIStatusString(ret)
       << " | Device #: " << dv_ind
       << " | Type: N/A"
       << " | Data: " << ((gfx_version == nullptr) ? "nullptr": std::to_string(*gfx_version));
    LOG_TRACE(ss);
    return ret;
    CATCH
}

enum iterator_handle_type {
  FUNC_ITER = 0,
  VARIANT_ITER,
  SUBVARIANT_ITER,
};

rsmi_status_t
rsmi_dev_supported_func_iterator_open(uint32_t dv_ind,
                                         rsmi_func_id_iter_handle_t *handle) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);
  GET_DEV_FROM_INDX

  if (handle == nullptr) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  dev->fillSupportedFuncs();

  *handle = new rsmi_func_id_iter_handle;

  if (*handle == nullptr) {
    return RSMI_STATUS_OUT_OF_RESOURCES;
  }

  (*handle)->id_type = FUNC_ITER;

  if (dev->supported_funcs()->begin() == dev->supported_funcs()->end()) {
    delete *handle;
    return RSMI_STATUS_NO_DATA;
  }

  SupportedFuncMapIt *supp_func_iter = new SupportedFuncMapIt;

  if (supp_func_iter == nullptr) {
    return RSMI_STATUS_OUT_OF_RESOURCES;
  }
  *supp_func_iter = dev->supported_funcs()->begin();

  (*handle)->func_id_iter = reinterpret_cast<uintptr_t>(supp_func_iter);
  (*handle)->container_ptr =
                        reinterpret_cast<uintptr_t>(dev->supported_funcs());

  return RSMI_STATUS_SUCCESS;

  CATCH
}

rsmi_status_t
rsmi_dev_supported_variant_iterator_open(
                                 rsmi_func_id_iter_handle_t parent_iter,
                                       rsmi_func_id_iter_handle_t *var_iter) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  if (var_iter == nullptr || parent_iter->id_type == SUBVARIANT_ITER) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  if (parent_iter->func_id_iter == 0) {
    return RSMI_STATUS_NO_DATA;
  }

  *var_iter = new rsmi_func_id_iter_handle;

  if (*var_iter == nullptr) {
    return RSMI_STATUS_OUT_OF_RESOURCES;
  }

  VariantMapIt *variant_itr = nullptr;
  SubVariantIt *sub_var_itr = nullptr;
  SupportedFuncMapIt *func_iter;
  std::shared_ptr<VariantMap> var_map_container;
  std::shared_ptr<SubVariant> sub_var_map_container;

  switch (parent_iter->id_type) {
    case FUNC_ITER:
      func_iter =
            reinterpret_cast<SupportedFuncMapIt *>(parent_iter->func_id_iter);
      var_map_container = (*func_iter)->second;

      if (var_map_container == nullptr) {
        delete *var_iter;
        return RSMI_STATUS_NO_DATA;
      }

      variant_itr = new VariantMapIt;
      *variant_itr = var_map_container->begin();
      (*var_iter)->func_id_iter = reinterpret_cast<uintptr_t>(variant_itr);
      (*var_iter)->container_ptr =
                         reinterpret_cast<uintptr_t>(var_map_container.get());
      (*var_iter)->id_type = VARIANT_ITER;
      break;

    case VARIANT_ITER:
      variant_itr =
                  reinterpret_cast<VariantMapIt *>(parent_iter->func_id_iter);
      sub_var_map_container = (*variant_itr)->second;

      if (sub_var_map_container == nullptr) {
        delete *var_iter;
        return RSMI_STATUS_NO_DATA;
      }

      sub_var_itr = new SubVariantIt;
      *sub_var_itr = sub_var_map_container->begin();
      (*var_iter)->func_id_iter = reinterpret_cast<uintptr_t>(sub_var_itr);
      (*var_iter)->container_ptr =
                     reinterpret_cast<uintptr_t>(sub_var_map_container.get());
      (*var_iter)->id_type = SUBVARIANT_ITER;
      break;

    default:
      assert(false);  // Unexpected iterator type
      return RSMI_STATUS_INVALID_ARGS;
  }
  return RSMI_STATUS_SUCCESS;

  CATCH
}

rsmi_status_t
rsmi_dev_supported_func_iterator_close(rsmi_func_id_iter_handle_t *handle) {
  TRY
  std::ostringstream ss;
  ss << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ss);

  if (handle == nullptr) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  if ((*handle)->id_type == FUNC_ITER) {
    SupportedFuncMapIt *supp_func_iter =
              reinterpret_cast<SupportedFuncMapIt *>((*handle)->func_id_iter);
    delete supp_func_iter;
  } else if ((*handle)->id_type == VARIANT_ITER) {
    VariantMapIt *var_iter =
                    reinterpret_cast<VariantMapIt *>((*handle)->func_id_iter);
    delete var_iter;
  }  else if ((*handle)->id_type == SUBVARIANT_ITER) {
    SubVariantIt *subvar_iter =
                      reinterpret_cast<SubVariantIt *>((*handle)->func_id_iter);
    delete subvar_iter;
  } else {
    return RSMI_STATUS_INVALID_ARGS;
  }

  delete *handle;

  *handle = nullptr;

  return RSMI_STATUS_SUCCESS;

  CATCH
}

rsmi_status_t
rsmi_func_iter_value_get(rsmi_func_id_iter_handle_t handle,
                                                rsmi_func_id_value_t *value) {
  TRY
  if (value == nullptr) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  if (handle->func_id_iter == 0) {
    return RSMI_STATUS_NO_DATA;
  }

  SupportedFuncMapIt *func_itr = nullptr;
  VariantMapIt *variant_itr = nullptr;
  SubVariantIt *sub_var_itr = nullptr;

  switch (handle->id_type) {
    case FUNC_ITER:
      func_itr = reinterpret_cast<SupportedFuncMapIt *>(handle->func_id_iter);
      value->name = (*func_itr)->first.c_str();
      break;

    case VARIANT_ITER:
      variant_itr = reinterpret_cast<VariantMapIt *>(handle->func_id_iter);
      value->id = (*variant_itr)->first;
      break;

    case SUBVARIANT_ITER:
      sub_var_itr = reinterpret_cast<SubVariantIt *>(handle->func_id_iter);

      // The hwmon file index that is appropriate for the rsmi user is stored
      // at bit position MONITOR_TYPE_BIT_POSITION.
      value->id = *(*sub_var_itr) >> MONITOR_TYPE_BIT_POSITION;
      break;

    default:
      return RSMI_STATUS_INVALID_ARGS;
  }
  CATCH
  return RSMI_STATUS_SUCCESS;
}

rsmi_status_t
rsmi_func_iter_next(rsmi_func_id_iter_handle_t handle) {
  TRY
  if (handle->func_id_iter == 0) {
    return RSMI_STATUS_NO_DATA;
  }

  SupportedFuncMapIt *func_iter;
  VariantMapIt *var_iter;
  SubVariantIt *sub_var_iter;

  switch (handle->id_type) {
    case FUNC_ITER:
      func_iter = reinterpret_cast<SupportedFuncMapIt *>(handle->func_id_iter);
      (*func_iter)++;

      if (*func_iter ==
         reinterpret_cast<SupportedFuncMap *>(handle->container_ptr)->end()) {
        return RSMI_STATUS_NO_DATA;
      }
      break;

    case VARIANT_ITER:
      var_iter = reinterpret_cast<VariantMapIt *>(handle->func_id_iter);
      (*var_iter)++;
      if (*var_iter ==
               reinterpret_cast<VariantMap *>(handle->container_ptr)->end()) {
        return RSMI_STATUS_NO_DATA;
      }
      break;

    case SUBVARIANT_ITER:
      sub_var_iter = reinterpret_cast<SubVariantIt *>(handle->func_id_iter);
      (*sub_var_iter)++;
      if (*sub_var_iter ==
               reinterpret_cast<SubVariant *>(handle->container_ptr)->end()) {
        return RSMI_STATUS_NO_DATA;
      }
      break;

    default:
      return RSMI_STATUS_INVALID_ARGS;
  }

  return RSMI_STATUS_SUCCESS;

  CATCH
}


static bool check_evt_notif_support(int kfd_fd) {
  struct kfd_ioctl_get_version_args args = {0, 0};

  if (ioctl(kfd_fd, AMDKFD_IOC_GET_VERSION, &args) == -1) {
    return false;
  }

  if (args.minor_version < 3) {
    return false;
  }
  return true;
}

static const char *kPathKFDIoctl = "/dev/kfd";

rsmi_status_t
rsmi_event_notification_init(uint32_t dv_ind) {
  TRY
  GET_DEV_FROM_INDX
  DEVICE_MUTEX

  std::lock_guard<std::mutex> guard(*smi.kfd_notif_evt_fh_mutex());
  if (smi.kfd_notif_evt_fh() == -1) {
    assert(smi.kfd_notif_evt_fh_refcnt() == 0);
    int kfd_fd = open(kPathKFDIoctl, O_RDWR | O_CLOEXEC);

    if (kfd_fd <= 0) {
      return RSMI_STATUS_FILE_ERROR;
    }

    if (!check_evt_notif_support(kfd_fd)) {
      close(kfd_fd);
      return RSMI_STATUS_NOT_SUPPORTED;
    }

    smi.set_kfd_notif_evt_fh(kfd_fd);
  }
  (void)smi.kfd_notif_evt_fh_refcnt_inc();
  struct kfd_ioctl_smi_events_args args;

  assert(dev->kfd_gpu_id() <= UINT32_MAX);
  args.gpuid = static_cast<uint32_t>(dev->kfd_gpu_id());

  int ret = ioctl(smi.kfd_notif_evt_fh(), AMDKFD_IOC_SMI_EVENTS, &args);
  if (ret < 0) {
    return amd::smi::ErrnoToRsmiStatus(errno);
  }
  if (args.anon_fd < 1) {
    return RSMI_STATUS_NO_DATA;
  }

  dev->set_evt_notif_anon_fd(args.anon_fd);
  FILE *anon_file_ptr = fdopen(static_cast<int>(args.anon_fd), "r");
  if (anon_file_ptr == nullptr) {
    close(dev->evt_notif_anon_fd());
    return amd::smi::ErrnoToRsmiStatus(errno);
  }
  dev->set_evt_notif_anon_file_ptr(anon_file_ptr);

  return RSMI_STATUS_SUCCESS;

  CATCH
}

rsmi_status_t
rsmi_event_notification_mask_set(uint32_t dv_ind, uint64_t mask) {
  TRY
  GET_DEV_FROM_INDX
  DEVICE_MUTEX

  if (dev->evt_notif_anon_fd() == -1) {
    return RSMI_INITIALIZATION_ERROR;
  }
  ssize_t ret = write(dev->evt_notif_anon_fd(), &mask, sizeof(uint64_t));

  if (ret == -1) {
    return amd::smi::ErrnoToRsmiStatus(errno);
  }

  return RSMI_STATUS_SUCCESS;
  CATCH
}

rsmi_status_t
rsmi_event_notification_get(int timeout_ms,
                     uint32_t *num_elem, rsmi_evt_notification_data_t *data) {
  TRY

  if (num_elem == nullptr || data == nullptr || *num_elem == 0) {
    return RSMI_STATUS_INVALID_ARGS;
  }

  uint32_t buffer_size = *num_elem;
  *num_elem = 0;

  rsmi_evt_notification_data_t *data_item;

  //  struct pollfd {
  //      int   fd;         /* file descriptor */
  //      short events;     /* requested events */
  //      short revents;    /* returned events */
  //  };
  std::vector<struct pollfd> fds;
  amd::smi::RocmSMI& smi = amd::smi::RocmSMI::getInstance();
  std::vector<uint32_t> fd_indx_to_dev_id;

  for (uint32_t i = 0; i < smi.devices().size(); ++i) {
    if (smi.devices()[i]->evt_notif_anon_fd() == -1) {
      continue;
    }
    fds.push_back({smi.devices()[i]->evt_notif_anon_fd(),
                                                     POLLIN | POLLRDNORM, 0});
    fd_indx_to_dev_id.push_back(i);
  }

  auto fill_data_buffer = [&](bool did_poll) {
    for (uint32_t i = 0; i < fds.size(); ++i) {
      if (did_poll) {
        if (!(fds[i].revents & (POLLIN | POLLRDNORM))) {
          continue;
        }
      }

      if (*num_elem >= buffer_size) {
        return;
      }

      FILE *anon_fp =
         smi.devices()[fd_indx_to_dev_id[i]]->evt_notif_anon_file_ptr();
      data_item =
           reinterpret_cast<rsmi_evt_notification_data_t *>(&data[*num_elem]);

      uint32_t event;
      while (fscanf(anon_fp, "%x %63s\n", &event,
                          reinterpret_cast<char *>(&data_item->message)) == 2) {
        /* Output is in format as "event information\n"
         * Both event are expressed in hex.
         * information is a string
         */
        data_item->event = (rsmi_evt_notification_type_t)event;
        data_item->dv_ind = fd_indx_to_dev_id[i];
        ++(*num_elem);

        if (*num_elem >= buffer_size) {
          break;
        }
        data_item =
             reinterpret_cast<rsmi_evt_notification_data_t *>(&data[*num_elem]);
      }
    }
  };

  // Collect any left-over events from a poll in a previous call to
  // rsmi_event_notification_get()
  fill_data_buffer(false);

  if (*num_elem < buffer_size && errno != EAGAIN) {
    return amd::smi::ErrnoToRsmiStatus(errno);
  }
  if (*num_elem >= buffer_size) {
    return RSMI_STATUS_SUCCESS;
  }

  // We still have buffer left, see if there are any new events
  int p_ret = poll(fds.data(), fds.size(), timeout_ms);
  if (p_ret > 0) {
    fill_data_buffer(true);
  } else if (p_ret < 0) {
    return amd::smi::ErrnoToRsmiStatus(errno);
  }
  if (*num_elem == 0) {
    return RSMI_STATUS_NO_DATA;
  }

  return RSMI_STATUS_SUCCESS;
  CATCH
}

rsmi_status_t rsmi_event_notification_stop(uint32_t dv_ind) {
  TRY
  GET_DEV_FROM_INDX
  DEVICE_MUTEX

  std::lock_guard<std::mutex> guard(*smi.kfd_notif_evt_fh_mutex());

  if (dev->evt_notif_anon_fd() == -1) {
    return RSMI_STATUS_INVALID_ARGS;
  }
//  close(dev->evt_notif_anon_fd());
  FILE *anon_fp = smi.devices()[dv_ind]->evt_notif_anon_file_ptr();
  fclose(anon_fp);
  assert(errno == 0 || errno == EAGAIN);
  dev->set_evt_notif_anon_file_ptr(nullptr);
  dev->set_evt_notif_anon_fd(-1);

  if (smi.kfd_notif_evt_fh_refcnt_dec() == 0) {
    int ret = close(smi.kfd_notif_evt_fh());
    smi.set_kfd_notif_evt_fh(-1);
    if (ret < 0) {
      return amd::smi::ErrnoToRsmiStatus(errno);
    }
  }

  return RSMI_STATUS_SUCCESS;
  CATCH
}

//
// NOTE:  APIs related to new 'GPU Metrics' related work are added here
//        so they can be used/tested.
//
rsmi_status_t
rsmi_dev_metrics_temp_edge_get(uint32_t dv_ind, uint16_t* edge_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(edge_value != nullptr);
  if (edge_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricTempEdge);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *edge_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_temp_hotspot_get(uint32_t dv_ind, uint16_t* hotspot_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(hotspot_value != nullptr);
  if (hotspot_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricTempHotspot);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *hotspot_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_temp_mem_get(uint32_t dv_ind, uint16_t* mem_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(mem_value != nullptr);
  if (mem_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricTempMem);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *mem_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_temp_vrgfx_get(uint32_t dv_ind, uint16_t* vrgfx_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(vrgfx_value != nullptr);
  if (vrgfx_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricTempVrGfx);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *vrgfx_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_temp_vrsoc_get(uint32_t dv_ind, uint16_t* vrsoc_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(vrsoc_value != nullptr);
  if (vrsoc_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricTempVrSoc);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *vrsoc_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_temp_vrmem_get(uint32_t dv_ind, uint16_t* vrmem_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(vrmem_value != nullptr);
  if (vrmem_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricTempVrMem);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *vrmem_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_curr_socket_power_get(uint32_t dv_ind, uint16_t* socket_power_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(socket_power_value != nullptr);
  if (socket_power_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricCurrSocketPower);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *socket_power_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_avg_socket_power_get(uint32_t dv_ind, uint16_t* socket_power_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(socket_power_value != nullptr);
  if (socket_power_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricAvgSocketPower);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *socket_power_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_avg_gfx_activity_get(uint32_t dv_ind, uint16_t* gfx_activity_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(gfx_activity_value != nullptr);
  if (gfx_activity_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricAvgGfxActivity);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *gfx_activity_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_avg_umc_activity_get(uint32_t dv_ind, uint16_t* umc_activity_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(umc_activity_value != nullptr);
  if (umc_activity_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricAvgUmcActivity);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *umc_activity_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_avg_mm_activity_get(uint32_t dv_ind, uint16_t* mm_activity_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(mm_activity_value != nullptr);
  if (mm_activity_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricAvgMmActivity);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *mm_activity_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_energy_acc_get(uint32_t dv_ind, uint64_t* energy_acc_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(energy_acc_value != nullptr);
  if (energy_acc_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricEnergyAccumulator);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *energy_acc_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_system_clock_counter_get(uint32_t dv_ind, uint64_t* system_clock_counter_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(system_clock_counter_value != nullptr);
  if (system_clock_counter_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricTSClockCounter);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *system_clock_counter_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_firmware_timestamp_get(uint32_t dv_ind, uint64_t* firmware_timestamp_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(firmware_timestamp_value != nullptr);
  if (firmware_timestamp_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricTSFirmware);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *firmware_timestamp_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_indep_throttle_status_get(uint32_t dv_ind, uint64_t* throttle_status_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(throttle_status_value != nullptr);
  if (throttle_status_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricIndepThrottleStatus);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *throttle_status_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_throttle_status_get(uint32_t dv_ind, uint32_t* throttle_status_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(throttle_status_value != nullptr);
  if (throttle_status_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricThrottleStatus);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *throttle_status_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_curr_fan_speed_get(uint32_t dv_ind, uint16_t* fan_speed_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(fan_speed_value != nullptr);
  if (fan_speed_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricCurrFanSpeed);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *fan_speed_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_pcie_link_width_get(uint32_t dv_ind, uint16_t* pcie_link_width_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(pcie_link_width_value != nullptr);
  if (pcie_link_width_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricPcieLinkWidth);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *pcie_link_width_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_pcie_link_speed_get(uint32_t dv_ind, uint16_t* pcie_link_speed_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(pcie_link_speed_value != nullptr);
  if (pcie_link_speed_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricPcieLinkSpeed);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *pcie_link_speed_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_xgmi_link_width_get(uint32_t dv_ind, uint16_t* xgmi_link_width_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(xgmi_link_width_value != nullptr);
  if (xgmi_link_width_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricXgmiLinkWidth);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *xgmi_link_width_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_xgmi_link_speed_get(uint32_t dv_ind, uint16_t* xgmi_link_speed_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(xgmi_link_speed_value != nullptr);
  if (xgmi_link_speed_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricXgmiLinkSpeed);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *xgmi_link_speed_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_gfxclk_lock_status_get(uint32_t dv_ind, uint32_t* gfxclk_lock_status_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(gfxclk_lock_status_value != nullptr);
  if (gfxclk_lock_status_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricGfxClkLockStatus);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *gfxclk_lock_status_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_gfx_activity_acc_get(uint32_t dv_ind, uint32_t* gfx_activity_acc_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(gfx_activity_acc_value != nullptr);
  if (gfx_activity_acc_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricGfxActivityAccumulator);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *gfx_activity_acc_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_mem_activity_acc_get(uint32_t dv_ind, uint32_t* mem_activity_acc_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(mem_activity_acc_value != nullptr);
  if (mem_activity_acc_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricMemActivityAccumulator);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *mem_activity_acc_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_pcie_bandwidth_acc_get(uint32_t dv_ind, uint64_t* pcie_bandwidth_acc_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(pcie_bandwidth_acc_value != nullptr);
  if (pcie_bandwidth_acc_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricPcieBandwidthAccumulator);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *pcie_bandwidth_acc_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_pcie_bandwidth_inst_get(uint32_t dv_ind, uint64_t* pcie_bandwidth_inst_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(pcie_bandwidth_inst_value != nullptr);
  if (pcie_bandwidth_inst_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricPcieBandwidthInst);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *pcie_bandwidth_inst_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_pcie_l0_recov_count_acc_get(uint32_t dv_ind, uint64_t* pcie_count_acc_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(pcie_count_acc_value != nullptr);
  if (pcie_count_acc_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricPcieL0RecovCountAccumulator);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *pcie_count_acc_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_pcie_replay_count_acc_get(uint32_t dv_ind, uint64_t* pcie_count_acc_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(pcie_count_acc_value != nullptr);
  if (pcie_count_acc_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricPcieReplayCountAccumulator);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *pcie_count_acc_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_pcie_replay_rover_count_acc_get(uint32_t dv_ind, uint64_t* pcie_count_acc_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(pcie_count_acc_value != nullptr);
  if (pcie_count_acc_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricPcieReplayRollOverCountAccumulator);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *pcie_count_acc_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_pcie_nak_sent_count_acc_get(uint32_t dv_ind, uint32_t* pcie_nak_sent_count_acc_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(pcie_nak_sent_count_acc_value != nullptr);
  if (pcie_nak_sent_count_acc_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricPcieNakSentCountAccumulator);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *pcie_nak_sent_count_acc_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_pcie_nak_rcvd_count_acc_get(uint32_t dv_ind, uint32_t* pcie_nak_rcvd_count_acc_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(pcie_nak_rcvd_count_acc_value != nullptr);
  if (pcie_nak_rcvd_count_acc_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricPcieNakReceivedCountAccumulator);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *pcie_nak_rcvd_count_acc_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_curr_uclk_get(uint32_t dv_ind, uint16_t* uclk_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(uclk_value != nullptr);
  if (uclk_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricCurrUClock);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *uclk_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_temp_hbm_get(uint32_t dv_ind, GPUMetricTempHbm_t* temp_hbm_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(temp_hbm_value != nullptr);
  if (temp_hbm_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricTempHbm);
  amd::smi::GPUMetricTempHbmTbl_t tmp_hbl_tbl{};
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, tmp_hbl_tbl);
  const auto max_num_elems =
    static_cast<uint16_t>(std::end(*temp_hbm_value) - std::begin(*temp_hbm_value));
  const auto copy_size =
    static_cast<uint16_t>((max_num_elems < tmp_hbl_tbl.size()) ? max_num_elems : tmp_hbl_tbl.size());
  ostrstream << __PRETTY_FUNCTION__
             << "\n | ======= end ======= "
             << "\n | End Result "
             << "\n | Device #:  " << dv_ind
             << "\n | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << "\n | Metric Size: " << tmp_hbl_tbl.size()
             << "\n | Max num of elements: " << max_num_elems
             << "\n | Copy size: " << copy_size
             << "\n | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);
  if (status_code == rsmi_status_t::RSMI_STATUS_SUCCESS) {
    std::memset(temp_hbm_value, 0, sizeof(*temp_hbm_value));
    std::copy_n(std::begin(tmp_hbl_tbl), copy_size, *temp_hbm_value);
  }

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_vcn_activity_get(uint32_t dv_ind, GPUMetricVcnActivity_t* vcn_activity_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(vcn_activity_value != nullptr);
  if (vcn_activity_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricVcnActivity);
  amd::smi::GPUMetricVcnActivityTbl_t tmp_vcn_tbl{};
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, tmp_vcn_tbl);
  const auto max_num_elems =
    static_cast<uint16_t>(std::end(*vcn_activity_value) - std::begin(*vcn_activity_value));
  const auto copy_size =
    static_cast<uint16_t>((max_num_elems < tmp_vcn_tbl.size()) ? max_num_elems : tmp_vcn_tbl.size());
  ostrstream << __PRETTY_FUNCTION__
             << "\n | ======= end ======= "
             << "\n | End Result "
             << "\n | Device #:  " << dv_ind
             << "\n | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << "\n | Metric Size: " << tmp_vcn_tbl.size()
             << "\n | Max num of elements: " << max_num_elems
             << "\n | Copy size: " << copy_size
             << "\n | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);
  if (status_code == rsmi_status_t::RSMI_STATUS_SUCCESS) {
    std::memset(vcn_activity_value, 0, sizeof(*vcn_activity_value));
    std::copy_n(std::begin(tmp_vcn_tbl), copy_size, *vcn_activity_value);
  }

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_jpeg_activity_get(uint32_t dv_ind, GPUMetricJpegActivity_t* jpeg_activity_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(jpeg_activity_value != nullptr);
  if (jpeg_activity_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricVcnActivity);
  amd::smi::GPUMetricJpegActivityTbl_t tmp_jpeg_tbl{};
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, tmp_jpeg_tbl);
  const auto max_num_elems =
    static_cast<uint16_t>(std::end(*jpeg_activity_value) - std::begin(*jpeg_activity_value));
  const auto copy_size =
    static_cast<uint16_t>((max_num_elems < tmp_jpeg_tbl.size()) ? max_num_elems : tmp_jpeg_tbl.size());
  ostrstream << __PRETTY_FUNCTION__
             << "\n | ======= end ======= "
             << "\n | End Result "
             << "\n | Device #:  " << dv_ind
             << "\n | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << "\n | Metric Size: " << tmp_jpeg_tbl.size()
             << "\n | Max num of elements: " << max_num_elems
             << "\n | Copy size: " << copy_size
             << "\n | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);
  if (status_code == rsmi_status_t::RSMI_STATUS_SUCCESS) {
    std::memset(jpeg_activity_value, 0, sizeof(*jpeg_activity_value));
    std::copy_n(std::begin(tmp_jpeg_tbl), copy_size, *jpeg_activity_value);
  }

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_xgmi_read_data_get(uint32_t dv_ind, GPUMetricXgmiReadDataAcc_t* xgmi_read_data_acc_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(xgmi_read_data_acc_value != nullptr);
  if (xgmi_read_data_acc_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricXgmiReadDataAccumulator);
  amd::smi::GPUMetricXgmiAccTbl_t tmp_xgmi_acc_tbl{};
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, tmp_xgmi_acc_tbl);
  const auto max_num_elems =
    static_cast<uint16_t>(std::end(*xgmi_read_data_acc_value) - std::begin(*xgmi_read_data_acc_value));
  const auto copy_size =
    static_cast<uint16_t>((max_num_elems < tmp_xgmi_acc_tbl.size()) ? max_num_elems : tmp_xgmi_acc_tbl.size());
  ostrstream << __PRETTY_FUNCTION__
             << "\n | ======= end ======= "
             << "\n | End Result "
             << "\n | Device #:  " << dv_ind
             << "\n | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << "\n | Metric Size: " << tmp_xgmi_acc_tbl.size()
             << "\n | Max num of elements: " << max_num_elems
             << "\n | Copy size: " << copy_size
             << "\n | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);
  if (status_code == rsmi_status_t::RSMI_STATUS_SUCCESS) {
    std::memset(xgmi_read_data_acc_value, 0, sizeof(*xgmi_read_data_acc_value));
    std::copy_n(std::begin(tmp_xgmi_acc_tbl), copy_size, *xgmi_read_data_acc_value);
  }

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_xgmi_write_data_get(uint32_t dv_ind, GPUMetricXgmiWriteDataAcc_t* xgmi_write_data_acc_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(xgmi_write_data_acc_value != nullptr);
  if (xgmi_write_data_acc_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricXgmiWriteDataAccumulator);
  amd::smi::GPUMetricXgmiAccTbl_t tmp_xgmi_acc_tbl{};
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, tmp_xgmi_acc_tbl);
  const auto max_num_elems =
    static_cast<uint16_t>(std::end(*xgmi_write_data_acc_value) - std::begin(*xgmi_write_data_acc_value));
  const auto copy_size =
    static_cast<uint16_t>((max_num_elems < tmp_xgmi_acc_tbl.size()) ? max_num_elems : tmp_xgmi_acc_tbl.size());
  ostrstream << __PRETTY_FUNCTION__
             << "\n | ======= end ======= "
             << "\n | End Result "
             << "\n | Device #:  " << dv_ind
             << "\n | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << "\n | Metric Size: " << tmp_xgmi_acc_tbl.size()
             << "\n | Max num of elements: " << max_num_elems
             << "\n | Copy size: " << copy_size
             << "\n | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);
  if (status_code == rsmi_status_t::RSMI_STATUS_SUCCESS) {
    std::memset(xgmi_write_data_acc_value, 0, sizeof(*xgmi_write_data_acc_value));
    std::copy_n(std::begin(tmp_xgmi_acc_tbl), copy_size, *xgmi_write_data_acc_value);
  }

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_curr_gfxclk_get(uint32_t dv_ind, GPUMetricCurrGfxClk_t* current_gfxclk_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(current_gfxclk_value != nullptr);
  if (current_gfxclk_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricCurrGfxClock);
  amd::smi::GPUMetricCurrGfxClkTbl_t tmp_curr_gfxclk_tbl{};
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, tmp_curr_gfxclk_tbl);
  const auto max_num_elems =
    static_cast<uint16_t>(std::end(*current_gfxclk_value) - std::begin(*current_gfxclk_value));
  const auto copy_size =
    static_cast<uint16_t>((max_num_elems < tmp_curr_gfxclk_tbl.size()) ? max_num_elems : tmp_curr_gfxclk_tbl.size());
  ostrstream << __PRETTY_FUNCTION__
             << "\n | ======= end ======= "
             << "\n | End Result "
             << "\n | Device #:  " << dv_ind
             << "\n | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << "\n | Metric Size: " << tmp_curr_gfxclk_tbl.size()
             << "\n | Max num of elements: " << max_num_elems
             << "\n | Copy size: " << copy_size
             << "\n | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);
  if (status_code == rsmi_status_t::RSMI_STATUS_SUCCESS) {
    std::memset(current_gfxclk_value, 0, sizeof(*current_gfxclk_value));
    std::copy_n(std::begin(tmp_curr_gfxclk_tbl), copy_size, *current_gfxclk_value);
  }

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_curr_socclk_get(uint32_t dv_ind, GPUMetricCurrSocClk_t* current_socclk_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(current_socclk_value != nullptr);
  if (current_socclk_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricCurrSocClock);
  amd::smi::GPUMetricCurrSocClkTbl_t tmp_curr_socclk_tbl{};
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, tmp_curr_socclk_tbl);
  const auto max_num_elems =
    static_cast<uint16_t>(std::end(*current_socclk_value) - std::begin(*current_socclk_value));
  const auto copy_size =
    static_cast<uint16_t>((max_num_elems < tmp_curr_socclk_tbl.size()) ? max_num_elems : tmp_curr_socclk_tbl.size());
  ostrstream << __PRETTY_FUNCTION__
             << "\n | ======= end ======= "
             << "\n | End Result "
             << "\n | Device #:  " << dv_ind
             << "\n | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << "\n | Metric Size: " << tmp_curr_socclk_tbl.size()
             << "\n | Max num of elements: " << max_num_elems
             << "\n | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);
  if (status_code == rsmi_status_t::RSMI_STATUS_SUCCESS) {
    std::memset(current_socclk_value, 0, sizeof(*current_socclk_value));
    std::copy_n(std::begin(tmp_curr_socclk_tbl), copy_size, *current_socclk_value);
  }

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_curr_vclk0_get(uint32_t dv_ind, GPUMetricCurrVClk0_t* current_vclk_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(current_vclk_value != nullptr);
  if (current_vclk_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricCurrVClock0);
  amd::smi::GPUMetricCurrVClkTbl_t tmp_curr_vclk0_tbl{};
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, tmp_curr_vclk0_tbl);
  const auto max_num_elems =
    static_cast<uint16_t>(std::end(*current_vclk_value) - std::begin(*current_vclk_value));
  const auto copy_size =
    static_cast<uint16_t>((max_num_elems < tmp_curr_vclk0_tbl.size()) ? max_num_elems : tmp_curr_vclk0_tbl.size());
  ostrstream << __PRETTY_FUNCTION__
             << "\n | ======= end ======= "
             << "\n | End Result "
             << "\n | Device #:  " << dv_ind
             << "\n | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << "\n | Metric Size: " << tmp_curr_vclk0_tbl.size()
             << "\n | Max num of elements: " << max_num_elems
             << "\n | Copy size: " << copy_size
             << "\n | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);
  if (status_code == rsmi_status_t::RSMI_STATUS_SUCCESS) {
    std::memset(current_vclk_value, 0, sizeof(*current_vclk_value));
    std::copy_n(std::begin(tmp_curr_vclk0_tbl), copy_size, *current_vclk_value);
  }

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_curr_vclk1_get(uint32_t dv_ind, uint16_t* current_vclk_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(current_vclk_value != nullptr);
  if (current_vclk_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricCurrVClock1);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *current_vclk_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_curr_dclk0_get(uint32_t dv_ind, GPUMetricCurrDClk0_t* current_dclk_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(current_dclk_value != nullptr);
  if (current_dclk_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricCurrDClock0);
  amd::smi::GPUMetricCurrDClkTbl_t tmp_curr_dclk0_tbl;
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, tmp_curr_dclk0_tbl);
  const auto max_num_elems =
    static_cast<uint16_t>(std::end(*current_dclk_value) - std::begin(*current_dclk_value));
  const auto copy_size =
    static_cast<uint16_t>((max_num_elems < tmp_curr_dclk0_tbl.size()) ? max_num_elems : tmp_curr_dclk0_tbl.size());
  ostrstream << __PRETTY_FUNCTION__
             << "\n | ======= end ======= "
             << "\n | End Result "
             << "\n | Device #:  " << dv_ind
             << "\n | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << "\n | Metric Size: " << tmp_curr_dclk0_tbl.size()
             << "\n | Max num of elements: " << max_num_elems
             << "\n | Copy size: " << copy_size
             << "\n | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);
  if (status_code == rsmi_status_t::RSMI_STATUS_SUCCESS) {
    std::memset(current_dclk_value, 0, sizeof(*current_dclk_value));
    std::copy_n(std::begin(tmp_curr_dclk0_tbl), copy_size, *current_dclk_value);
  }

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_curr_dclk1_get(uint32_t dv_ind, uint16_t* current_dclk_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(current_dclk_value != nullptr);
  if (current_dclk_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricCurrDClock1);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *current_dclk_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_avg_gfx_clock_frequency_get(uint32_t dv_ind, uint16_t* clock_frequency_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(clock_frequency_value != nullptr);
  if (clock_frequency_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricAvgGfxClockFrequency);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *clock_frequency_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_avg_soc_clock_frequency_get(uint32_t dv_ind, uint16_t* clock_frequency_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(clock_frequency_value != nullptr);
  if (clock_frequency_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricAvgSocClockFrequency);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *clock_frequency_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_avg_uclock_frequency_get(uint32_t dv_ind, uint16_t* clock_frequency_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(clock_frequency_value != nullptr);
  if (clock_frequency_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricAvgUClockFrequency);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *clock_frequency_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_avg_vclock0_frequency_get(uint32_t dv_ind, uint16_t* clock_frequency_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(clock_frequency_value != nullptr);
  if (clock_frequency_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricAvgVClock0Frequency);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *clock_frequency_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_avg_dclock0_frequency_get(uint32_t dv_ind, uint16_t* clock_frequency_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(clock_frequency_value != nullptr);
  if (clock_frequency_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricAvgDClock0Frequency);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *clock_frequency_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_avg_vclock1_frequency_get(uint32_t dv_ind, uint16_t* clock_frequency_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(clock_frequency_value != nullptr);
  if (clock_frequency_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricAvgVClock1Frequency);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *clock_frequency_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_avg_dclock1_frequency_get(uint32_t dv_ind, uint16_t* clock_frequency_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(clock_frequency_value != nullptr);
  if (clock_frequency_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricAvgDClock1Frequency);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *clock_frequency_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_volt_soc_get(uint32_t dv_ind, uint16_t* voltage_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(voltage_value != nullptr);
  if (voltage_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricVoltageSoc);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *voltage_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_volt_gfx_get(uint32_t dv_ind, uint16_t* voltage_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(voltage_value != nullptr);
  if (voltage_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricVoltageGfx);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *voltage_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_volt_mem_get(uint32_t dv_ind, uint16_t* voltage_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(voltage_value != nullptr);
  if (voltage_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricVoltageMem);
  auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, *voltage_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_header_info_get(uint32_t dv_ind, metrics_table_header_t* header_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(header_value != nullptr);
  if (header_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  auto status_code = rsmi_dev_gpu_metrics_header_info_get(dv_ind, *header_value);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Format Revision: " << header_value->format_revision
             << " | Content Revision: " << header_value->content_revision
             << " | Header Size: " << header_value->structure_size
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}


rsmi_status_t
rsmi_dev_metrics_xcd_counter_get(uint32_t dv_ind, uint16_t* xcd_counter_value)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  assert(xcd_counter_value != nullptr);
  if (xcd_counter_value == nullptr) {
    return rsmi_status_t::RSMI_STATUS_INVALID_ARGS;
  }

  auto xcd_counter = uint16_t(0);
  GPUMetricCurrGfxClk_t curr_gfxclk_table{};
  auto status_code = rsmi_dev_metrics_curr_gfxclk_get(dv_ind, &curr_gfxclk_table);
  if (status_code == rsmi_status_t::RSMI_STATUS_SUCCESS) {
    for (const auto& gfxclk : curr_gfxclk_table) {
      if ((gfxclk != 0) && (gfxclk != UINT16_MAX)) {
        xcd_counter++;
      }
    }
  }

  *xcd_counter_value = xcd_counter;
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | XCDs counter: " << xcd_counter
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

rsmi_status_t
rsmi_dev_metrics_log_get(uint32_t dv_ind)
{
  TRY
  std::ostringstream ostrstream;
  ostrstream << __PRETTY_FUNCTION__ << "| ======= start =======";
  LOG_TRACE(ostrstream);

  GET_DEV_FROM_INDX
  auto status_code = dev->dev_log_gpu_metrics(ostrstream);
  ostrstream << __PRETTY_FUNCTION__
             << " | ======= end ======= "
             << " | End Result "
             << " | Device #:  " << dv_ind
             << " | Metric Type: " << "All GPU Metrics..."
             << " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
  LOG_INFO(ostrstream);

  return status_code;
  CATCH
}

//
// End of: new GPU Metrics related work.
//


// UNDOCUMENTED FUNCTIONS
// This functions are not declared in rocm_smi.h. They are either not fully
// supported, or to be used for test purposes.

// This function acquires a mutex and waits for a number of seconds
rsmi_status_t
rsmi_test_sleep(uint32_t dv_ind, uint32_t seconds) {
//  DEVICE_MUTEX
  amd::smi::pthread_wrap _pw(*amd::smi::GetMutex(dv_ind));
  amd::smi::RocmSMI& smi_ = amd::smi::RocmSMI::getInstance();
  bool blocking_ = !(smi_.init_options() &
                      static_cast<uint64_t>(RSMI_INIT_FLAG_RESRV_TEST1));
  amd::smi::ScopedPthread _lock(_pw, blocking_);
  if (!blocking_ && _lock.mutex_not_acquired()) {
    return RSMI_STATUS_BUSY;
  }

  sleep(seconds);
  return RSMI_STATUS_SUCCESS;
}

int32_t
rsmi_test_refcount(uint64_t refcnt_type) {
  (void)refcnt_type;

  amd::smi::RocmSMI& smi = amd::smi::RocmSMI::getInstance();
  std::lock_guard<std::mutex> guard(*smi.bootstrap_mutex());

  if (smi.ref_count() == 0 && !smi.devices().empty()) {
    return -1;
  }

  return static_cast<int32_t>(smi.ref_count());
}