Event counter support

XGMI related events are supported

Change-Id: If17036fe890c8be45da3654353599821b5828c14

CMakeLists.txt

@@ -89,6 +89,7 @@ set(SMI_SRC_LIST ${SMI_SRC_LIST} "${SRC_DIR}/rocm_smi_monitor.cc")
 set(SMI_SRC_LIST ${SMI_SRC_LIST} "${SRC_DIR}/rocm_smi.cc")
 set(SMI_SRC_LIST ${SMI_SRC_LIST} "${SRC_DIR}/rocm_smi_power_mon.cc")
 set(SMI_SRC_LIST ${SMI_SRC_LIST} "${SRC_DIR}/rocm_smi_utils.cc")
+set(SMI_SRC_LIST ${SMI_SRC_LIST} "${SRC_DIR}/rocm_smi_counters.cc")
 set(SMI_SRC_LIST ${SMI_SRC_LIST} "${SRC_DIR}/shared_mutex/shared_mutex.c")
 
 set(SMI_INC_LIST "${INC_DIR}/rocm_smi_device.h")
@@ -98,6 +99,7 @@ set(SMI_INC_LIST ${SMI_INC_LIST} "${INC_DIR}/rocm_smi_power_mon.h")
 set(SMI_INC_LIST ${SMI_INC_LIST} "${INC_DIR}/rocm_smi_utils.h")
 set(SMI_INC_LIST ${SMI_INC_LIST} "${INC_DIR}/rocm_smi_common.h")
 set(SMI_INC_LIST ${SMI_INC_LIST} "${INC_DIR}/rocm_smi_exception.h")
+set(SMI_INC_LIST ${SMI_INC_LIST} "${INC_DIR}/rocm_smi_counters.h")
 set(SMI_INC_LIST ${SMI_INC_LIST} "${SRC_DIR}/shared_mutex/shared_mutex.h")
 
 set(SMI_EXAMPLE_EXE "rocm_smi_ex")

include/rocm_smi/rocm_smi.h

@@ -109,7 +109,11 @@ typedef enum {
   RSMI_STATUS_NOT_FOUND,                 //!< An item was searched for but not
                                          //!< found
   RSMI_STATUS_INSUFFICIENT_SIZE,         //!< Not enough resources were
-                                         //!< for the operation
+                                         //!< available for the operation
+  RSMI_STATUS_INTERRUPT,                 //!< An interrupt occurred during
+                                         //!< execution of function
+  RSMI_STATUS_UNEXPECTED_SIZE,           //!< An unexpected amount of data
+                                         //!< was read
   RSMI_STATUS_UNKNOWN_ERROR = 0xFFFFFFFF,  //!< An unknown error occurred
 } rsmi_status_t;
 
@@ -171,6 +175,74 @@ typedef enum {
   RSMI_SW_COMP_LAST = RSMI_SW_COMP_DRIVER
 } rsmi_sw_component_t;
 
+/**
+ * Event counter types
+ */
+
+/**
+ * @brief Handle to performance event counter
+ */
+typedef uintptr_t rsmi_event_handle_t;
+
+/**
+ * Event Groups
+ *
+ * @brief Enum denoting an event group. The value of the enum is the
+ * base value for all the event enums in the group.
+ */
+typedef enum {
+  RSMI_EVNT_GRP_XGMI = 0,         //!< Data Fabric (XGMI) related events
+
+  RSMI_EVNT_GRP_INVALID = 0xFFFFFFFF
+} rsmi_event_group_t;
+
+/**
+ * Event types
+ * @brief Event type enum. Events belonging to a particular event group
+ * ::rsmi_event_group_t should begin ennumerating at the ::rsmi_event_group_t
+ * value for that group.
+ */
+typedef enum {
+  RSMI_EVNT_FIRST = RSMI_EVNT_GRP_XGMI,
+
+  RSMI_EVNT_XGMI_FIRST = RSMI_EVNT_GRP_XGMI,
+  RSMI_EVNT_XGMI_0_NOP_TX = RSMI_EVNT_XGMI_FIRST,  //!< NOPs sent to neighbor 0
+  RSMI_EVNT_XGMI_0_REQUEST_TX,                    //!< Outgoing requests to
+                                                  //!< neighbor 0
+  RSMI_EVNT_XGMI_0_RESPONSE_TX,                   //!< Outgoing responses to
+                                                  //!< neighbor 0
+  RSMI_EVNT_XGMI_0_BEATS_TX,                      //!< Data beats sent to
+                                                  //!< neighbor 0
+  RSMI_EVNT_XGMI_1_NOP_TX,                        //!< NOPs sent to neighbor 1
+  RSMI_EVNT_XGMI_1_REQUEST_TX,                        //!< Outgoing requests to
+                                                  //!< neighbor 1
+  RSMI_EVNT_XGMI_1_RESPONSE_TX,                   //!< Outgoing responses to
+                                                  //!< neighbor 1
+  RSMI_EVNT_XGMI_1_BEATS_TX,                      //!< Data beats sent to
+                                                  //!< neighbor 1
+
+  RSMI_EVNT_XGMI_LAST = RSMI_EVNT_XGMI_1_BEATS_TX,
+
+  RSMI_EVNT_LAST = RSMI_EVNT_XGMI_LAST
+} rsmi_event_type_t;
+
+/**
+ * Event counter commands
+ */
+typedef enum {
+  RSMI_CNTR_CMD_START = 0,  //!< Start the counter
+  RSMI_CNTR_CMD_STOP,       //!< Stop the counter
+} rsmi_counter_command_t;
+
+/**
+ * Counter value
+ */
+typedef struct {
+  uint64_t value;            //!< Counter value
+  uint64_t time_enabled;     //!< Time that the counter was enabled
+  uint64_t time_running;     //!< Time that che counter was running
+} rsmi_counter_value_t;
+
 /**
  * Clock types
  */
@@ -876,9 +948,8 @@ rsmi_status_t rsmi_dev_pci_bandwidth_set(uint32_t dv_ind, uint64_t bw_bitmask);
  *  device index.
  *
  *  @details Given a device index @p dv_ind and a pointer to a uint64_t
- *  @p power, this function will write the current average power consumption to
- *  the uint64_t in microwatts pointed to by @p power. This function requires
- *  root privilege.
+ *  @p power, this function will write the current average power consumption
+ *  (in microwatts) to the uint64_t pointed to by @p power.
  *
  *  @param[in] dv_ind a device index
  *
@@ -1077,16 +1148,13 @@ rsmi_status_t rsmi_dev_fan_rpms_get(uint32_t dv_ind, uint32_t sensor_ind,
                                                               int64_t *speed);
 
 /**
- * @brief Get the fan speed for the specified device in RPMs.
- *
- * @details Given a device index @p dv_ind
- * this function will get the fan speed.
- *
- * @param[in] dv_ind a device index
+ * @brief Get the fan speed for the specified device as a value relative to
+ * ::RSMI_MAX_FAN_SPEED
  *
  * @details Given a device index @p dv_ind and a pointer to a uint32_t
  * @p speed, this function will write the current fan speed (a value
- * between 0 and 255) to the uint32_t pointed to by @p speed
+ * between 0 and the maximum fan speed, ::RSMI_MAX_FAN_SPEED) to the uint32_t
+ * pointed to by @p speed
  *
  * @param[in] dv_ind a device index
  *
@@ -1268,7 +1336,8 @@ rsmi_status_t rsmi_dev_overdrive_level_get(uint32_t dv_ind, uint32_t *od);
  *  @param[in] clk_type the type of clock for which the frequency is desired
  *
  *  @param[inout] f a pointer to a caller provided ::rsmi_frequencies_t structure
- *  to which the frequency information will be written
+ *  to which the frequency information will be written. Frequency values are in
+ *  Hz.
  *
  *  @retval ::RSMI_STATUS_SUCCESS is returned upon successful call.
  *
@@ -1628,6 +1697,134 @@ rsmi_status_string(rsmi_status_t status, const char **status_string);
 
 /** @} */  // end of ErrQuer
 
+/*****************************************************************************/
+/** @defgroup PerfCntr Performance Counter Functions
+ *  These functions are used to configure, query and control performance
+ *  counting.
+ *  @{
+ */
+
+/**
+ * @brief Tell if an event group is supported by a given device
+ *
+ * @details Given a device index @p dv_ind and an event group specifier @p
+ * group, tell if @p group type events are supported by the device associated
+ * with @p dv_ind
+ *
+ * @param[in] dv_ind device index of device being queried
+ *
+ * @param[in] group ::rsmi_event_group_t identifier of group for which support
+ * is being queried
+ *
+ * @retval
+ * ::RSMI_STATUS_SUCCESS if the device associatee with @p dv_ind
+ * support counting events of the type indicated by @p group.
+ *
+ * ::RSMI_STATUS_NOT_SUPPORTED If the device does not support event group @p
+ * group
+ *
+ */
+rsmi_status_t
+rsmi_dev_counter_group_supported(uint32_t dv_ind, rsmi_event_group_t group);
+
+/**
+ * @brief Create a performance counter object
+ *
+ * @details Create a performance counter object of type @p type for the device
+ * with a device index of @p dv_ind, and write a handle to the object to the
+ * memory location pointed to by @p evnt_handle. @p evnt_handle can be used
+ * with other performance event operations. The handle should be deallocated
+ * with ::rsmi_dev_counter_destroy() when no longer needed.
+ *
+ * @param[in] dv_ind a device index
+ *
+ * @param[in] type the type of performance event to create
+ *
+ * @param[inout] evnt_handle A pointer to a ::rsmi_event_handle_t which will be
+ * associated with a newly allocated counter
+ *
+ * @retval ::RSMI_STATUS_SUCCESS is returned upon successful call
+ *
+ */
+rsmi_status_t
+rsmi_dev_counter_create(uint32_t dv_ind, rsmi_event_type_t type,
+                                            rsmi_event_handle_t *evnt_handle);
+
+/**
+ * @brief Deallocate a performance counter object
+ *
+ * @details Deallocate the performance counter object with the provided
+ * ::rsmi_event_handle_t @p evnt_handle
+ *
+ * @param[in] evnt_handle handle to event object to be deallocated
+ *
+ * @retval ::RSMI_STATUS_SUCCESS is returned upon successful call
+ *
+ */
+rsmi_status_t
+rsmi_dev_counter_destroy(rsmi_event_handle_t evnt_handle);
+
+/**
+ * @brief Issue performance counter control commands
+ *
+ * @details Issue a command @p cmd on the event counter associated with the
+ * provided handle @p evt_handle.
+ *
+ * @param[in] evt_handle an event handle
+ *
+ * @param[in] cmd The event counter command to be issued
+ *
+ * @param[inout] cmd_args Currently not used. Should be set to NULL.
+ *
+ * @retval ::RSMI_STATUS_SUCCESS is returned upon successful call
+ *
+ */
+rsmi_status_t
+rsmi_counter_control(rsmi_event_handle_t evt_handle,
+                                  rsmi_counter_command_t cmd, void *cmd_args);
+
+/**
+ * @brief Read the current value of a performance counter
+ *
+ * @details Read the current counter value of the counter associated with the
+ * provided handle @p evt_handle and write the value to the location pointed
+ * to by @p value.
+ *
+ * @param[in] evt_handle an event handle
+ *
+ * @param[inout] value pointer to memory of size of ::rsmi_counter_value_t to
+ * which the counter value will be written
+ *
+ * @retval ::RSMI_STATUS_SUCCESS is returned upon successful call
+ *
+ */
+rsmi_status_t
+rsmi_counter_read(rsmi_event_handle_t evt_handle,
+                                                 rsmi_counter_value_t *value);
+
+/**
+ * @brief Get the number of currently available counters
+ *
+ * @details Given a device index @p dv_ind, a performance event group @p grp,
+ * and a pointer to a uint32_t @p available, this function will write the
+ * number of @p grp type counters that are available on the device with index
+ * @p dv_ind to the memory that @p available points to.
+ *
+ * @param[in] dv_ind a device index
+ *
+ * @param[in] grp an event device group
+ *
+ * @param[inout] available A pointer to a uint32_t to which the number of
+ * available counters will be written
+ *
+ * @retval ::RSMI_STATUS_SUCCESS is returned upon successful call
+ *
+ */
+rsmi_status_t
+rsmi_counter_available_counters_get(uint32_t dv_ind,
+                                 rsmi_event_group_t grp, uint32_t *available);
+/** @} */  // end of PerfCntr
+
 #ifdef __cplusplus
 }
 #endif  // __cplusplus

include/rocm_smi/rocm_smi_counters.h

@@ -0,0 +1,124 @@
+/*
+ * =============================================================================
+ *   ROC Runtime Conformance Release License
+ * =============================================================================
+ * The University of Illinois/NCSA
+ * Open Source License (NCSA)
+ *
+ * Copyright (c) 2019, Advanced Micro Devices, Inc.
+ * All rights reserved.
+ *
+ * Developed by:
+ *
+ *                 AMD Research and AMD ROC Software Development
+ *
+ *                 Advanced Micro Devices, Inc.
+ *
+ *                 www.amd.com
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal with the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ *  - Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimers.
+ *  - Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimers in
+ *    the documentation and/or other materials provided with the distribution.
+ *  - Neither the names of <Name of Development Group, Name of Institution>,
+ *    nor the names of its contributors may be used to endorse or promote
+ *    products derived from this Software without specific prior written
+ *    permission.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS WITH THE SOFTWARE.
+ *
+ */
+#ifndef INCLUDE_ROCM_SMI_ROCM_SMI_COUNTERS_H_
+#define INCLUDE_ROCM_SMI_ROCM_SMI_COUNTERS_H_
+
+#include <linux/perf_event.h>
+
+#include <cstdint>
+#include <vector>
+#include <unordered_set>
+#include <string>
+
+#include "rocm_smi/rocm_smi.h"
+
+namespace amd {
+namespace smi {
+namespace evt {
+
+class RSMIEventGrpHashFunction {
+ public:
+    size_t operator()(const rsmi_event_group_t& grp) const {
+      return static_cast<size_t>(grp);
+    }
+};
+
+typedef std::unordered_set<rsmi_event_group_t, RSMIEventGrpHashFunction>
+                                                            dev_evt_grp_set_t;
+void
+GetSupportedEventGroups(uint32_t dev_ind, dev_evt_grp_set_t*supported_grps);
+
+struct evnt_info_t {
+    uint8_t start_bit;
+    uint8_t field_size;
+    uint32_t value;
+};
+
+struct perf_read_format_t {
+    union {
+        struct {
+            uint64_t value;
+            uint64_t enabled_time;
+            uint64_t run_time;
+        };
+        uint64_t values[3];
+    };
+};
+
+class Event {
+ public:
+    explicit Event(rsmi_event_type_t event, uint32_t dev_ind);
+    ~Event(void);
+
+    uint32_t openPerfHandle();
+    uint32_t startCounter(void);
+    uint32_t stopCounter(void);
+    uint32_t getValue(rsmi_counter_value_t *val);
+    uint32_t dev_file_ind(void) const {return dev_file_ind_;}
+    uint32_t dev_ind(void) const {return dev_ind_;}
+
+ private:
+    // perf_event_attr fields
+    std::vector<evnt_info_t> event_info_;
+
+    std::string evt_path_root_;
+
+    rsmi_event_type_t event_type_;
+    uint32_t dev_file_ind_;
+    uint32_t dev_ind_;
+    int32_t fd_;
+    perf_event_attr attr_;
+
+    uint32_t get_event_file_info(void);
+    uint32_t get_event_type(uint32_t *ev_type);
+};
+
+
+}  // namespace evt
+}  // namespace smi
+}  // namespace amd
+
+#endif  // INCLUDE_ROCM_SMI_ROCM_SMI_COUNTERS_H_
+

include/rocm_smi/rocm_smi_device.h

@@ -50,11 +50,13 @@
 #include <utility>
 #include <cstdint>
 #include <vector>
+#include <unordered_set>
 
 #include "rocm_smi/rocm_smi_monitor.h"
 #include "rocm_smi/rocm_smi_power_mon.h"
 #include "rocm_smi/rocm_smi_common.h"
 #include "rocm_smi/rocm_smi.h"
+#include "rocm_smi/rocm_smi_counters.h"
 extern "C" {
 #include "shared_mutex.h"   // NOLINT
 };
@@ -92,6 +94,7 @@ enum DevInfoTypes {
   kDevMemUsedVRAM,
   kDevPCIEReplayCount,
   kDevUniqueId,
+  kDevDFCountersAvailable,
 };
 
 class Device {
@@ -118,6 +121,8 @@ class Device {
     void set_bdfid(uint64_t val) {bdfid_ = val;}
     uint64_t get_bdfid(void) const {return bdfid_;}
     pthread_mutex_t *mutex(void) {return mutex_.ptr;}
+    evt::dev_evt_grp_set_t* supported_event_groups(void) {
+                                             return &supported_event_groups_;}
 
  private:
     std::shared_ptr<Monitor> monitor_;
@@ -134,6 +139,8 @@ class Device {
                                             std::vector<std::string> *retVec);
     int writeDevInfoStr(DevInfoTypes type, std::string valStr);
     uint64_t bdfid_;
+    std::unordered_set<rsmi_event_group_t,
+                       evt::RSMIEventGrpHashFunction> supported_event_groups_;
 };
 
 }  // namespace smi

include/rocm_smi/rocm_smi_exception.h

@@ -55,7 +55,7 @@ namespace smi {
 /// @brief Exception type which carries an error code to return to the user.
 class rsmi_exception : public std::exception {
  public:
-  rsmi_exception(rsmi_status_t error, const char* description) :
+  rsmi_exception(rsmi_status_t error, const std::string description) :
                                             err_(error), desc_(description) {}
   rsmi_status_t error_code() const noexcept { return err_; }
   const char* what() const noexcept override { return desc_.c_str(); }

src/rocm_smi.cc

@@ -61,6 +61,7 @@
 #include "rocm_smi/rocm_smi_device.h"
 #include "rocm_smi/rocm_smi_utils.h"
 #include "rocm_smi/rocm_smi_exception.h"
+#include "rocm_smi/rocm_smi_counters.h"
 
 #include "rocm_smi/rocm_smi64Config.h"
 
@@ -93,11 +94,15 @@ static rsmi_status_t handleException() {
 
 #define TRY try {
 #define CATCH } catch (...) {return handleException();}
-#define GET_DEV_FROM_INDX  \
+
+#define CHECK_DV_IND_RANGE \
     amd::smi::RocmSMI& smi = amd::smi::RocmSMI::getInstance(); \
     if (dv_ind >= smi.monitor_devices().size()) { \
       return RSMI_STATUS_INVALID_ARGS; \
     } \
+
+#define GET_DEV_FROM_INDX  \
+  CHECK_DV_IND_RANGE \
   std::shared_ptr<amd::smi::Device> dev = smi.monitor_devices()[dv_ind]; \
   assert(dev != nullptr);
 
@@ -128,7 +133,10 @@ static rsmi_status_t errno_to_rsmi_status(uint32_t err) {
     case EACCES: return RSMI_STATUS_PERMISSION;
     case EPERM:
     case ENOENT: return RSMI_STATUS_NOT_SUPPORTED;
+    case EBADF:
     case EISDIR: return RSMI_STATUS_FILE_ERROR;
+    case EINTR:  return RSMI_STATUS_INTERRUPT;
+    case EIO:    return RSMI_STATUS_UNEXPECTED_SIZE;
     default:     return RSMI_STATUS_UNKNOWN_ERROR;
   }
 }
@@ -1970,6 +1978,10 @@ rsmi_status_string(rsmi_status_t status, const char **status_string) {
                           " successfully";
       break;
 
+    case RSMI_STATUS_INTERRUPT:
+      *status_string = "An interrupt occurred while executing the function";
+      break;
+
     default:
       *status_string = "An unknown error occurred";
       return RSMI_STATUS_UNKNOWN_ERROR;
@@ -2118,3 +2130,153 @@ rsmi_dev_unique_id_get(uint32_t dv_ind, uint64_t *unique_id) {
 
   CATCH
 }
+rsmi_status_t
+rsmi_dev_counter_create(uint32_t dv_ind, rsmi_event_type_t type,
+                                           rsmi_event_handle_t *evnt_handle) {
+  TRY
+  DEVICE_MUTEX
+  REQUIRE_ROOT_ACCESS
+  CHECK_DV_IND_RANGE
+
+  if (evnt_handle == nullptr) {
+    return RSMI_STATUS_INVALID_ARGS;
+  }
+  if (type < RSMI_EVNT_FIRST || type > RSMI_EVNT_LAST) {
+    return RSMI_STATUS_INVALID_ARGS;
+  }
+
+  *evnt_handle = reinterpret_cast<uintptr_t>(
+                                      new amd::smi::evt::Event(type, dv_ind));
+
+  if (evnt_handle == nullptr) {
+    return RSMI_STATUS_OUT_OF_RESOURCES;
+  }
+
+  return RSMI_STATUS_SUCCESS;
+  CATCH
+}
+
+rsmi_status_t
+rsmi_dev_counter_destroy(rsmi_event_handle_t evnt_handle) {
+  TRY
+
+  if (evnt_handle == 0) {
+    return RSMI_STATUS_INVALID_ARGS;
+  }
+
+  amd::smi::evt::Event *evt =
+                        reinterpret_cast<amd::smi::evt::Event *>(evnt_handle);
+  uint32_t dv_ind = evt->dev_ind();
+  DEVICE_MUTEX
+  REQUIRE_ROOT_ACCESS
+
+  delete evt;
+  return RSMI_STATUS_SUCCESS;
+  CATCH
+}
+
+rsmi_status_t
+rsmi_counter_control(rsmi_event_handle_t evt_handle,
+                                 rsmi_counter_command_t cmd, void *cmd_args) {
+  TRY
+
+  amd::smi::evt::Event *evt =
+                         reinterpret_cast<amd::smi::evt::Event *>(evt_handle);
+  amd::smi::pthread_wrap _pw(*get_mutex(evt->dev_ind()));
+  amd::smi::ScopedPthread _lock(_pw);
+
+  REQUIRE_ROOT_ACCESS
+
+  uint32_t ret;
+
+  // This is for future command args. This would work in conjunction with a
+  // new function to set perf attributes.
+  (void) cmd_args;
+
+  if (evt_handle == 0) {
+    return RSMI_STATUS_INVALID_ARGS;
+  }
+
+  switch (cmd) {
+    case RSMI_CNTR_CMD_START:
+      ret = evt->startCounter();
+      break;
+
+    case RSMI_CNTR_CMD_STOP:
+      ret = evt->stopCounter();
+      break;
+
+    default:
+      assert(!"Unexpected perf counter command");
+  }
+  return errno_to_rsmi_status(ret);
+
+  CATCH
+}
+
+rsmi_status_t
+rsmi_counter_read(rsmi_event_handle_t evt_handle,
+                                                rsmi_counter_value_t *value) {
+  TRY
+
+  if (value == nullptr || evt_handle == 0) {
+    return RSMI_STATUS_INVALID_ARGS;
+  }
+
+  amd::smi::evt::Event *evt =
+                         reinterpret_cast<amd::smi::evt::Event *>(evt_handle);
+
+  uint32_t dv_ind = evt->dev_ind();
+  DEVICE_MUTEX
+  REQUIRE_ROOT_ACCESS
+
+  uint32_t ret;
+
+  ret = evt->getValue(value);
+
+  return errno_to_rsmi_status(ret);
+  CATCH
+}
+
+rsmi_status_t
+rsmi_counter_available_counters_get(uint32_t dv_ind,
+                                rsmi_event_group_t grp, uint32_t *available) {
+  rsmi_status_t ret;
+
+  TRY
+  if (available == nullptr) {
+    return RSMI_STATUS_INVALID_ARGS;
+  }
+  DEVICE_MUTEX
+  uint64_t val;
+
+  switch (grp) {
+    case RSMI_EVNT_GRP_XGMI:
+      ret = get_dev_value_int(amd::smi::kDevDFCountersAvailable, dv_ind, &val);
+      assert(val < UINT32_MAX);
+      *available = static_cast<uint32_t>(val);
+      break;
+
+    default:
+      return RSMI_STATUS_INVALID_ARGS;
+  }
+  return ret;
+  CATCH
+}
+
+rsmi_status_t
+rsmi_dev_counter_group_supported(uint32_t dv_ind, rsmi_event_group_t group) {
+  TRY
+  DEVICE_MUTEX
+  GET_DEV_FROM_INDX
+
+  amd::smi::evt::dev_evt_grp_set_t *grp = dev->supported_event_groups();
+
+  if (grp->find(group) == grp->end()) {
+    return RSMI_STATUS_NOT_SUPPORTED;
+  } else {
+    return RSMI_STATUS_SUCCESS;
+  }
+  CATCH
+}
+

src/rocm_smi_counters.cc

@@ -0,0 +1,407 @@
+/*
+ * =============================================================================
+ * The University of Illinois/NCSA
+ * Open Source License (NCSA)
+ *
+ * Copyright (c) 2019, Advanced Micro Devices, Inc.
+ * All rights reserved.
+ *
+ * Developed by:
+ *
+ *                 AMD Research and AMD ROC Software Development
+ *
+ *                 Advanced Micro Devices, Inc.
+ *
+ *                 www.amd.com
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal with the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ *  - Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimers.
+ *  - Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimers in
+ *    the documentation and/or other materials provided with the distribution.
+ *  - Neither the names of <Name of Development Group, Name of Institution>,
+ *    nor the names of its contributors may be used to endorse or promote
+ *    products derived from this Software without specific prior written
+ *    permission.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS WITH THE SOFTWARE.
+ *
+ */
+
+#include <assert.h>
+#include <string.h>
+#include <linux/perf_event.h>
+#include <unistd.h>
+#include <asm/unistd.h>
+#include <sys/ioctl.h>
+#include <sys/stat.h>
+#include <stdio.h>
+
+#include <map>
+#include <sstream>
+#include <algorithm>
+#include <iostream>
+#include <fstream>
+#include <unordered_set>
+
+#include "rocm_smi/rocm_smi.h"
+#include "rocm_smi/rocm_smi_counters.h"
+#include "rocm_smi/rocm_smi_utils.h"
+#include "rocm_smi/rocm_smi_exception.h"
+#include "rocm_smi/rocm_smi_main.h"
+#include "rocm_smi/rocm_smi_device.h"
+
+namespace amd {
+namespace smi {
+namespace evt {
+
+static const char *kPathDeviceEventRoot = "/sys/bus/event_source/devices";
+
+// Event group names
+static const char *kEvGrpDataFabricFName = "amdgpu_df_#";
+
+// Data Fabric event file names
+static const char *kDFEvtCake0FtiReqAllocFName = "cake0_ftiinstat_reqalloc";
+static const char *kDFEvtCake0FtiRspAllocFName = "cake0_ftiinstat_rspalloc";
+static const char *kDFEvtCake0PcsOutTxDataFName = "cake0_pcsout_txdata";
+static const char *kDFEvtCake0PcsOutTxMetaFName = "cake0_pcsout_txmeta";
+static const char *kDFEvtCake1FtiReqAllocFName = "cake1_ftiinstat_reqalloc";
+static const char *kDFEvtCake1FtiRspAllocFName = "cake1_ftiinstat_rspalloc";
+static const char *kDFEvtCake1PcsOutTxDataFName = "cake1_pcsout_txdata";
+static const char *kDFEvtCake1PcsOutTxMetaFName = "cake1_pcsout_txmeta";
+
+
+static const std::map<rsmi_event_type_t, const char *> kEventFNameMap = {
+  {RSMI_EVNT_XGMI_0_NOP_TX,      kDFEvtCake0PcsOutTxMetaFName},
+  {RSMI_EVNT_XGMI_0_REQUEST_TX,  kDFEvtCake0FtiReqAllocFName},
+  {RSMI_EVNT_XGMI_0_RESPONSE_TX, kDFEvtCake0FtiRspAllocFName},
+  {RSMI_EVNT_XGMI_0_BEATS_TX,    kDFEvtCake0PcsOutTxDataFName},
+  {RSMI_EVNT_XGMI_1_NOP_TX,      kDFEvtCake1PcsOutTxMetaFName},
+  {RSMI_EVNT_XGMI_1_REQUEST_TX,  kDFEvtCake1FtiReqAllocFName},
+  {RSMI_EVNT_XGMI_1_RESPONSE_TX, kDFEvtCake1FtiRspAllocFName},
+  {RSMI_EVNT_XGMI_1_BEATS_TX,    kDFEvtCake1PcsOutTxDataFName},
+};
+
+static const std::map<rsmi_event_group_t, const char *> kEvtGrpFNameMap = {
+    {RSMI_EVNT_GRP_XGMI, kEvGrpDataFabricFName},
+    {RSMI_EVNT_GRP_INVALID, "bogus"},
+};
+
+static rsmi_event_group_t EvtGrpFromEvtID(rsmi_event_type_t evnt) {
+#define EVNT_GRP_RANGE_CHK(EVGRP_SHORT, EVGRP_ENUM) \
+  if (evnt >= RSMI_EVNT_##EVGRP_SHORT##_FIRST && \
+                                   evnt <= RSMI_EVNT_##EVGRP_SHORT##_LAST) { \
+    return EVGRP_ENUM; \
+  }
+  EVNT_GRP_RANGE_CHK(XGMI, RSMI_EVNT_GRP_XGMI);
+
+  return RSMI_EVNT_GRP_INVALID;
+}
+
+// Note below that dev_num is not the same as the usual dv_ind.
+// dev_num is the number of the device (e.g., 1 for card1) whereas dv_ind
+// is usually the index into the vector of devices
+void
+GetSupportedEventGroups(uint32_t dev_num, dev_evt_grp_set_t *supported_grps) {
+  assert(supported_grps != nullptr);
+
+  std::string grp_path_base;
+  std::string grp_path;
+  uint32_t ret;
+
+  grp_path_base = kPathDeviceEventRoot;
+  grp_path_base += '/';
+  struct stat file_stat;
+
+  for (auto g : kEvtGrpFNameMap) {
+    grp_path = grp_path_base;
+    grp_path += g.second;
+
+    std::replace(grp_path.begin(), grp_path.end(), '#',
+                                            static_cast<char>('0' + dev_num));
+
+    ret = stat(grp_path.c_str(), &file_stat);
+
+    if (ret) {
+      assert(errno == ENOENT);
+      continue;
+    }
+    if (S_ISDIR(file_stat.st_mode)) {
+      supported_grps->insert(g.first);
+    }
+  }
+}
+//  /sys/bus/event_source/devices/<hw block>_<instance>/type
+Event::Event(rsmi_event_type_t event, uint32_t dev_ind)  :
+                                       event_type_(event) {
+  fd_ = -1;
+  rsmi_event_group_t grp = EvtGrpFromEvtID(event);
+  assert(grp != RSMI_EVNT_GRP_INVALID);  // This should have failed before now
+
+  evt_path_root_ = kPathDeviceEventRoot;
+  evt_path_root_ += '/';
+  evt_path_root_ += kEvtGrpFNameMap.at(grp);
+
+
+  amd::smi::RocmSMI& smi = amd::smi::RocmSMI::getInstance();
+  assert(dev_ind < smi.monitor_devices().size());
+  std::shared_ptr<amd::smi::Device> dev = smi.monitor_devices()[dev_ind];
+  assert(dev != nullptr);
+
+
+  dev_ind_ = dev_ind;
+  dev_file_ind_ = dev->index();
+  std::replace(evt_path_root_.begin(), evt_path_root_.end(), '#',
+                                      static_cast<char>('0' + dev_file_ind_));
+}
+Event::~Event(void) {
+  int ret;
+  if (fd_ != -1) {
+    ret = close(fd_);
+
+    if (ret == -1) {
+      perror("Failed to close file descriptor.");
+    }
+  }
+}
+
+static void
+parse_field_config(std::string fstr, evnt_info_t *val) {
+  std::stringstream ss(fstr);
+  std::stringstream fs;
+  std::string config_ln;
+  std::string field_name;
+  uint32_t start_bit;
+  uint32_t end_bit;
+  char jnk;
+
+  assert(val != nullptr);
+
+  getline(ss, config_ln, ':');
+  ss >> start_bit;
+  ss >> jnk;
+  assert(jnk == '-');
+  ss >> end_bit;
+
+  val->start_bit = start_bit;
+  val->field_size = end_bit - start_bit + 1;
+}
+
+static uint32_t
+get_event_bitfield_info(std::string *config_path, evnt_info_t *val) {
+  uint32_t err;
+
+  std::string fstr;
+
+  err = ReadSysfsStr(*config_path, &fstr);
+  if (err) {
+    return err;
+  }
+  parse_field_config(fstr, val);
+
+  return 0;
+}
+
+uint32_t
+Event::get_event_file_info(void) {
+  uint32_t err;
+
+  std::string fn = evt_path_root_;
+  std::string fstr;
+
+  fn += "/events/";
+  fn += kEventFNameMap.at(event_type_);
+  err = ReadSysfsStr(fn, &fstr);
+  if (err) {
+    return err;
+  }
+  // parse_perf_attr(fstr, &event_id_.event_field_vals);
+  std::stringstream ss(fstr);
+  std::stringstream fs;
+  std::string field_assgn;
+  std::string field_name;
+  evnt_info_t ev_info;
+
+  while (ss.rdbuf()->in_avail() != 0) {
+    ev_info = {};
+    getline(ss, field_assgn, ',');
+    fs.clear();
+    fs << field_assgn;
+    getline(fs, field_name, '=');
+    fs >> std::hex >> ev_info.value;
+    assert(fs.rdbuf()->in_avail() == 0);
+
+    // Now, get the corresponding bitfield
+    std::string config_path = evt_path_root_;
+    config_path += "/format/";
+    config_path += field_name;
+    err = get_event_bitfield_info(&config_path, &ev_info);
+    if (err) {
+      return err;
+    }
+    event_info_.push_back(ev_info);
+  }
+  return 0;
+}
+
+uint32_t
+Event::get_event_type(uint32_t *ev_type) {
+  assert(ev_type != nullptr);
+  if (ev_type == nullptr) {
+    return EINVAL;
+  }
+  std::string fn = evt_path_root_;
+  std::string fstr;
+
+  fn += "/type";
+
+  std::ifstream fs;
+  fs.open(fn);
+
+  if (!fs.is_open()) {
+    return errno;
+  }
+  fs >> *ev_type;
+  fs.close();
+  return 0;
+}
+
+static uint64_t
+get_perf_attr_config(std::vector<evnt_info_t> *ev_info) {
+  uint64_t ret_val = 0;
+
+  assert(ev_info != nullptr);
+
+  for (const evnt_info_t& ev : *ev_info) {
+    ret_val |= ev.value << ev.start_bit;
+  }
+  return ret_val;
+}
+
+uint32_t
+amd::smi::evt::Event::openPerfHandle(void) {
+  uint32_t ret;
+
+  memset(&attr_, 0, sizeof(struct perf_event_attr));
+
+  ret = get_event_file_info();
+  if (ret) {
+    return ret;
+  }
+  ret = get_event_type(&attr_.type);
+  if (ret) {
+    return ret;
+  }
+
+  attr_.size = sizeof(struct perf_event_attr);
+  attr_.config = get_perf_attr_config(&event_info_);
+  attr_.read_format = PERF_FORMAT_TOTAL_TIME_ENABLED |
+                          PERF_FORMAT_TOTAL_TIME_RUNNING;
+  attr_.disabled = 1;
+  attr_.inherit = 1;
+
+  fd_ = syscall(__NR_perf_event_open, &attr_,
+                           -1, 0, -1, PERF_FLAG_FD_NO_GROUP);
+
+  if (fd_ < 0) {
+    return errno;
+  }
+  return 0;
+}
+
+uint32_t
+amd::smi::evt::Event::startCounter(void) {
+  int32_t ret;
+
+  if (fd_ == -1) {
+    ret = openPerfHandle();
+    if (ret != 0) {
+      return ret;
+    }
+  }
+  ret = ioctl(fd_, PERF_EVENT_IOC_ENABLE, NULL);
+
+  if (ret == -1) {
+    return errno;
+  }
+
+  assert(ret == 0);  // We're expecting the ioctl call to return -1 or 0
+  return 0;
+}
+
+uint32_t
+amd::smi::evt::Event::stopCounter(void) {
+  int32_t ret;
+
+  if (fd_ == -1) {
+    return EBADF;
+  }
+  ret = ioctl(fd_, PERF_EVENT_IOC_DISABLE, NULL);
+
+  if (ret == -1) {
+    return errno;
+  }
+
+  assert(ret == 0);  // We're expecting the ioctl call to return -1 or 0
+  return 0;
+}
+
+static ssize_t
+readn(int fd, void *buf, size_t n) {
+  size_t left = n;
+  ssize_t bytes;
+
+  while (left) {
+    bytes = read(fd, buf, left);
+    if (!bytes) /* reach EOF */
+      return (n - left);
+    if (bytes < 0) {
+      if (errno == EINTR) /* read got interrupted */
+        continue;
+      else
+        return -errno;
+    }
+    left -= bytes;
+    buf = reinterpret_cast<void *>((reinterpret_cast<uint8_t *>(buf) + bytes));
+  }
+  return n;
+}
+
+uint32_t
+amd::smi::evt::Event::getValue(rsmi_counter_value_t *val) {
+  assert(val != nullptr);
+  ssize_t ret;
+
+  perf_read_format_t pvalue;
+  ret = readn(fd_, &pvalue, sizeof(perf_read_format_t));
+  if (ret < 0) {
+    return -ret;
+  }
+
+  if (ret != sizeof(perf_read_format_t)) {
+    return EIO;
+  }
+
+  val->value = pvalue.value;
+  val->time_enabled = pvalue.enabled_time;
+  val->time_running = pvalue.run_time;
+
+  return 0;
+}
+
+}  // namespace evt
+}  // namespace smi
+}  // namespace amd

src/rocm_smi_device.cc

@@ -98,6 +98,7 @@ static const char *kDevMemUsedVisVRAMFName = "mem_info_vis_vram_used";
 static const char *kDevMemUsedVRAMFName = "mem_info_vram_used";
 static const char *kDevPCIEReplayCountFName = "pcie_replay_count";
 static const char *kDevUniqueIdFName = "unique_id";
+static const char *kDevDFCountersAvailableFName = "df_cntr_avail";
 
 // Strings that are found within sysfs files
 static const char *kDevPerfLevelAutoStr = "auto";
@@ -140,6 +141,7 @@ static const std::map<DevInfoTypes, const char *> kDevAttribNameMap = {
     {kDevMemUsedVRAM, kDevMemUsedVRAMFName},
     {kDevPCIEReplayCount, kDevPCIEReplayCountFName},
     {kDevUniqueId, kDevUniqueIdFName},
+    {kDevDFCountersAvailable, kDevDFCountersAvailableFName},
 };
 
 static const std::map<rsmi_dev_perf_level, const char *> kDevPerfLvlMap = {
@@ -155,10 +157,18 @@ static const std::map<rsmi_dev_perf_level, const char *> kDevPerfLvlMap = {
     {RSMI_DEV_PERF_LEVEL_UNKNOWN, kDevPerfLevelUnknownStr},
 };
 
-static bool isRegularFile(std::string fname) {
+static int isRegularFile(std::string fname, bool *is_reg) {
   struct stat file_stat;
-  stat(fname.c_str(), &file_stat);
-  return S_ISREG(file_stat.st_mode);
+  int ret;
+
+  assert(is_reg != nullptr);
+
+  ret = stat(fname.c_str(), &file_stat);
+  if (ret) {
+    return errno;
+  }
+  *is_reg = S_ISREG(file_stat.st_mode);
+  return 0;
 }
 
 #define RET_IF_NONZERO(X) { \
@@ -205,7 +215,14 @@ int Device::openSysfsFileStream(DevInfoTypes type, T *fs, const char *str) {
   sysfs_path += kDevAttribNameMap.at(type);
 
   DBG_FILE_ERROR(sysfs_path, str);
-  if (!isRegularFile(sysfs_path)) {
+  bool reg_file;
+
+  int ret = isRegularFile(sysfs_path, &reg_file);
+
+  if (ret != 0) {
+    return ret;
+  }
+  if (!reg_file) {
     return ENOENT;
   }
 
@@ -372,6 +389,7 @@ int Device::readDevInfo(DevInfoTypes type, uint64_t *val) {
     case kDevMemUsedVisVRAM:
     case kDevMemUsedVRAM:
     case kDevPCIEReplayCount:
+    case kDevDFCountersAvailable:
       ret = readDevInfoStr(type, &tempStr);
       RET_IF_NONZERO(ret);
       *val = std::stoul(tempStr, 0);

src/rocm_smi_main.cc

@@ -58,6 +58,7 @@
 #include <cerrno>
 
 #include "rocm_smi/rocm_smi.h"
+#include "rocm_smi/rocm_smi_device.h"
 #include "rocm_smi/rocm_smi_main.h"
 #include "rocm_smi/rocm_smi_exception.h"
 
@@ -332,6 +333,7 @@ RocmSMI::AddToDeviceList(std::string dev_name) {
   uint32_t d_index = GetDeviceIndex(d_name);
   dev->set_index(d_index);
 
+  GetSupportedEventGroups(d_index, dev->supported_event_groups());
   devices_.push_back(dev);
 
   return;

tests/rocm_smi_test/functional/perf_cntr_read_write.cc

@@ -0,0 +1,304 @@
+/*
+ * =============================================================================
+ *   ROC Runtime Conformance Release License
+ * =============================================================================
+ * The University of Illinois/NCSA
+ * Open Source License (NCSA)
+ *
+ * Copyright (c) 2019, Advanced Micro Devices, Inc.
+ * All rights reserved.
+ *
+ * Developed by:
+ *
+ *                 AMD Research and AMD ROC Software Development
+ *
+ *                 Advanced Micro Devices, Inc.
+ *
+ *                 www.amd.com
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal with the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ *  - Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimers.
+ *  - Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimers in
+ *    the documentation and/or other materials provided with the distribution.
+ *  - Neither the names of <Name of Development Group, Name of Institution>,
+ *    nor the names of its contributors may be used to endorse or promote
+ *    products derived from this Software without specific prior written
+ *    permission.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS WITH THE SOFTWARE.
+ *
+ */
+
+#include <stdint.h>
+#include <stddef.h>
+
+#include <iostream>
+#include <bitset>
+#include <string>
+#include <algorithm>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "rocm_smi/rocm_smi.h"
+#include "rocm_smi_test/functional/perf_cntr_read_write.h"
+#include "rocm_smi_test/test_common.h"
+
+PerfCntrEvtGrp::PerfCntrEvtGrp(rsmi_event_group_t grp, uint32_t first,
+        uint32_t last, std::string name) : grp_(grp), first_evt_(first),
+                                                last_evt_(last), name_(name) {
+  num_events_ = last_evt_ - first_evt_ + 1;
+}
+
+PerfCntrEvtGrp::~PerfCntrEvtGrp() {}
+
+// Add new event groups to test here
+#define PC_EVT_GRP(SHRT, NAME) \
+    PerfCntrEvtGrp(RSMI_EVNT_GRP_##SHRT, RSMI_EVNT_##SHRT##_FIRST, \
+                                                RSMI_EVNT_##SHRT##_LAST, NAME)
+static const std::vector<PerfCntrEvtGrp> s_event_groups = {
+    PC_EVT_GRP(XGMI, "XGMI")
+};
+
+TestPerfCntrReadWrite::TestPerfCntrReadWrite() : TestBase() {
+  set_title("RSMI Performance Counter Read/Write Test");
+  set_description("The Performance counter tests verify that performance"
+                            " counters can be controlled and read properly.");
+}
+
+TestPerfCntrReadWrite::~TestPerfCntrReadWrite(void) {
+}
+
+void TestPerfCntrReadWrite::SetUp(void) {
+  TestBase::SetUp();
+
+  return;
+}
+
+void TestPerfCntrReadWrite::DisplayTestInfo(void) {
+  TestBase::DisplayTestInfo();
+}
+
+void TestPerfCntrReadWrite::DisplayResults(void) const {
+  TestBase::DisplayResults();
+  return;
+}
+
+void TestPerfCntrReadWrite::Close() {
+  // This will close handles opened within rsmitst utility calls and call
+  // rsmi_shut_down(), so it should be done after other hsa cleanup
+  TestBase::Close();
+}
+
+#define RSMI_EVNT_ENUM_FIRST(GRP_NAME) RSMI_EVNT_##GRP_NAME##_FIRST
+#define RSMI_EVNT_ENUM_LAST(GRP_NAME) RSMI_EVNT_##GRP_NAME##_LAST
+
+// Refactor this to handle different event groups once we have > 1 event group
+void
+TestPerfCntrReadWrite::testEventsIndividually(uint32_t dv_ind) {
+  rsmi_event_handle_t evt_handle;
+  rsmi_status_t ret;
+  rsmi_counter_value_t val;
+
+  std::cout << "****************************" << std::endl;
+  std::cout << "Test each event individually" << std::endl;
+  std::cout << "****************************" << std::endl;
+
+  for (PerfCntrEvtGrp grp : s_event_groups) {
+    if (rsmi_dev_counter_group_supported(dv_ind, grp.group())
+                                               == RSMI_STATUS_NOT_SUPPORTED) {
+      continue;
+    }
+
+    IF_VERB(STANDARD) {
+      std::cout << "Testing Event Group " << grp.name() << std::endl;
+    }
+
+    for (uint32_t evnt = grp.first_evt(); evnt <= grp.last_evt(); ++evnt) {
+      IF_VERB(STANDARD) {
+        std::cout << "\tTesting Event Type " << evnt << std::endl;
+      }
+
+      IF_VERB(STANDARD) {
+        std::cout << "\t\tCreating event..." << std::endl;
+      }
+      ret = rsmi_dev_counter_create(dv_ind,
+                           static_cast<rsmi_event_type_t>(evnt), &evt_handle);
+      CHK_ERR_ASRT(ret)
+
+      IF_VERB(STANDARD) {
+        std::cout << "\t\tStart Counting..." << std::endl;
+      }
+      ret = rsmi_counter_control(evt_handle, RSMI_CNTR_CMD_START, nullptr);
+      CHK_ERR_ASRT(ret)
+
+      sleep(1);
+      IF_VERB(STANDARD) {
+        std::cout << "\t\tStop Counting..." << std::endl;
+      }
+      ret = rsmi_counter_control(evt_handle, RSMI_CNTR_CMD_STOP, nullptr);
+      CHK_ERR_ASRT(ret)
+
+      IF_VERB(STANDARD) {
+        std::cout << "\t\tRead Counter..." << std::endl;
+      }
+      ret = rsmi_counter_read(evt_handle, &val);
+      CHK_ERR_ASRT(ret)
+
+      IF_VERB(STANDARD) {
+        std::cout << "\t\tSuccessfully read value: " << std::endl;
+        std::cout << "\t\t\tValue: " << val.value << std::endl;
+        std::cout << "\t\t\tTime Enabled: " << val.time_enabled << std::endl;
+        std::cout << "\t\t\tTime Running: " << val.time_running << std::endl;
+      }
+      IF_VERB(STANDARD) {
+        std::cout << "\t\tRe-start Counting..." << std::endl;
+      }
+      ret = rsmi_counter_control(evt_handle, RSMI_CNTR_CMD_START, nullptr);
+      CHK_ERR_ASRT(ret)
+
+      IF_VERB(STANDARD) {
+        std::cout << "\t\tRead free-running Counter..." << std::endl;
+      }
+      ret = rsmi_counter_read(evt_handle, &val);
+      CHK_ERR_ASRT(ret)
+
+      IF_VERB(STANDARD) {
+        std::cout << "\t\tSuccessfully read value: " << std::endl;
+        std::cout << "\t\t\tValue: " << val.value << std::endl;
+        std::cout << "\t\t\tTime Enabled: " << val.time_enabled << std::endl;
+        std::cout << "\t\t\tTime Running: " << val.time_running << std::endl;
+      }
+      ret = rsmi_dev_counter_destroy(evt_handle);
+      CHK_ERR_ASRT(ret)
+    }
+  }
+}
+
+// Refactor this to handle different event groups once we have > 1 event group
+void
+TestPerfCntrReadWrite::testEventsSimultaneously(uint32_t dv_ind) {
+  rsmi_event_handle_t evt_handle[RSMI_EVNT_XGMI_LAST -
+                                                    RSMI_EVNT_XGMI_FIRST + 1];
+  rsmi_status_t ret;
+  rsmi_counter_value_t val;
+  uint32_t avail_counters;
+
+  std::cout << "****************************" << std::endl;
+  std::cout << "Test events simultaneously"   << std::endl;
+  std::cout << "****************************" << std::endl;
+
+  for (PerfCntrEvtGrp grp : s_event_groups) {
+    if (rsmi_dev_counter_group_supported(dv_ind, grp.group())
+                                               == RSMI_STATUS_NOT_SUPPORTED) {
+      std::cout << "\tEvent Group" << grp.name() <<
+                                  " is not supported. Skipping." << std::endl;
+      continue;
+    }
+    IF_VERB(STANDARD) {
+      std::cout << "Testing Event Group " << grp.name() << std::endl;
+    }
+
+    ret = rsmi_counter_available_counters_get(dv_ind, grp.group(),
+                                                               &avail_counters);
+    std::cout << "Available Counters: " << avail_counters << std::endl;
+
+    CHK_ERR_ASRT(ret)
+
+    uint32_t tmp;
+
+    for (uint32_t evnt = grp.first_evt(); evnt <= grp.last_evt();
+                                                     evnt += avail_counters) {
+      IF_VERB(STANDARD) {
+        std::cout << "\tTesting Event Type " << evnt << std::endl;
+      }
+      IF_VERB(STANDARD) {
+        std::cout << "\tCreating events..." << std::endl;
+      }
+      for (uint32_t j = 0; j < avail_counters; ++j) {
+        tmp = static_cast<rsmi_event_type_t>(evnt + j);
+        IF_VERB(STANDARD) {
+          std::cout << "\tEvent Type " << tmp << std::endl;
+        }
+
+        ret = rsmi_dev_counter_create(dv_ind,
+                     static_cast<rsmi_event_type_t>(tmp), &evt_handle[tmp]);
+        CHK_ERR_ASRT(ret)
+      }
+
+      IF_VERB(STANDARD) {
+        std::cout << "\tStart Counters..." << std::endl;
+      }
+      uint32_t tmp_cntrs;
+
+      for (uint32_t j = 0; j < avail_counters; ++j) {
+        tmp = static_cast<rsmi_event_type_t>(evnt + j);
+        ret = rsmi_counter_control(evt_handle[tmp], RSMI_CNTR_CMD_START,
+                                                                     nullptr);
+        CHK_ERR_ASRT(ret)
+
+        ret = rsmi_counter_available_counters_get(dv_ind, grp.group(),
+                                                                  &tmp_cntrs);
+        CHK_ERR_ASRT(ret)
+        ASSERT_EQ(tmp_cntrs, (avail_counters - j -1));
+      }
+
+      sleep(5);
+      IF_VERB(STANDARD) {
+        std::cout << "\tStop Counters..." << std::endl;
+      }
+      for (uint32_t j = 0; j < avail_counters; ++j) {
+        tmp = static_cast<rsmi_event_type_t>(evnt + j);
+        ret = rsmi_counter_control(evt_handle[tmp], RSMI_CNTR_CMD_STOP,
+                                                                     nullptr);
+        CHK_ERR_ASRT(ret)
+      }
+
+      IF_VERB(STANDARD) {
+        std::cout << "\tRead Counters..." << std::endl;
+      }
+      for (uint32_t j = 0; j < avail_counters; ++j) {
+        tmp = static_cast<rsmi_event_type_t>(evnt + j);
+        ret = rsmi_counter_read(evt_handle[tmp], &val);
+        CHK_ERR_ASRT(ret)
+
+        IF_VERB(STANDARD) {
+          std::cout << "\tCounter: " << tmp << std::endl;
+          std::cout << "\tSuccessfully read value: " << std::endl;
+          std::cout << "\t\tValue: " << val.value << std::endl;
+          std::cout << "\t\tTime Enabled: " << val.time_enabled << std::endl;
+          std::cout << "\t\tTime Running: " << val.time_running << std::endl;
+        }
+      }
+      for (uint32_t j = 0; j < avail_counters; ++j) {
+        tmp = static_cast<rsmi_event_type_t>(evnt + j);
+        ret = rsmi_dev_counter_destroy(evt_handle[tmp]);
+        CHK_ERR_ASRT(ret)
+      }
+    }
+  }
+}
+
+void TestPerfCntrReadWrite::Run(void) {
+  TestBase::Run();
+
+  for (uint32_t dv_ind = 0; dv_ind < num_monitor_devs(); ++dv_ind) {
+    PrintDeviceHeader(dv_ind);
+
+    testEventsIndividually(dv_ind);
+    testEventsSimultaneously(dv_ind);
+  }
+}

tests/rocm_smi_test/functional/perf_cntr_read_write.h

@@ -0,0 +1,98 @@
+/*
+ * =============================================================================
+ *   ROC Runtime Conformance Release License
+ * =============================================================================
+ * The University of Illinois/NCSA
+ * Open Source License (NCSA)
+ *
+ * Copyright (c) 2019, Advanced Micro Devices, Inc.
+ * All rights reserved.
+ *
+ * Developed by:
+ *
+ *                 AMD Research and AMD ROC Software Development
+ *
+ *                 Advanced Micro Devices, Inc.
+ *
+ *                 www.amd.com
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal with the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ *  - Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimers.
+ *  - Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimers in
+ *    the documentation and/or other materials provided with the distribution.
+ *  - Neither the names of <Name of Development Group, Name of Institution>,
+ *    nor the names of its contributors may be used to endorse or promote
+ *    products derived from this Software without specific prior written
+ *    permission.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS WITH THE SOFTWARE.
+ *
+ */
+#ifndef TESTS_ROCM_SMI_TEST_FUNCTIONAL_PERF_CNTR_READ_WRITE_H_
+#define TESTS_ROCM_SMI_TEST_FUNCTIONAL_PERF_CNTR_READ_WRITE_H_
+
+#include <string>
+
+#include "rocm_smi_test/test_base.h"
+
+class TestPerfCntrReadWrite : public TestBase {
+ public:
+    TestPerfCntrReadWrite();
+
+  // @Brief: Destructor for test case of TestPerfCntrReadWrite
+  virtual ~TestPerfCntrReadWrite();
+
+  // @Brief: Setup the environment for measurement
+  virtual void SetUp();
+
+  // @Brief: Core measurement execution
+  virtual void Run();
+
+  // @Brief: Clean up and retrive the resource
+  virtual void Close();
+
+  // @Brief: Display  results
+  virtual void DisplayResults() const;
+
+  // @Brief: Display information about what this test does
+  virtual void DisplayTestInfo(void);
+
+ private:
+  void testEventsIndividually(uint32_t dv_ind);
+  void testEventsSimultaneously(uint32_t dv_ind);
+};
+
+class PerfCntrEvtGrp {
+ public:
+    explicit PerfCntrEvtGrp(rsmi_event_group_t grp,
+                             uint32_t first, uint32_t last, std::string name);
+    ~PerfCntrEvtGrp();
+
+    rsmi_event_group_t group(void) const { return grp_;}
+    uint32_t first_evt(void) const {return first_evt_;}
+    uint32_t last_evt(void) const {return last_evt_;}
+    uint32_t num_events(void) const {return num_events_;}
+    std::string name(void) const { return name_;}
+ private:
+    rsmi_event_group_t grp_;
+    uint32_t first_evt_;
+    uint32_t last_evt_;
+    uint32_t num_events_;
+    std::string name_;
+};
+
+#endif  // TESTS_ROCM_SMI_TEST_FUNCTIONAL_PERF_CNTR_READ_WRITE_H_

tests/rocm_smi_test/main.cc

@@ -72,6 +72,7 @@
 #include "functional/err_cnt_read.h"
 #include "functional/mem_util_read.h"
 #include "functional/id_info_read.h"
+#include "rocm_smi_test/functional/perf_cntr_read_write.h"
 
 static RSMITstGlobals *sRSMIGlvalues = nullptr;
 
@@ -198,7 +199,10 @@ TEST(rsmitstReadOnly, TestIdInfoRead) {
   TestIdInfoRead tst;
   RunGenericTest(&tst);
 }
-
+TEST(rsmitstreadWrite,TestPerfCntrReadWrite) {
+  TestPerfCntrReadWrite tst;
+  RunGenericTest(&tst);
+}
 int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);