From 0e17236bc5d6056ea72663ddce70dafb35eed78b Mon Sep 17 00:00:00 2001
From: Harish Kasiviswanathan <Harish.Kasiviswanathan@amd.com>
Date: Tue, 4 May 2021 21:18:27 -0400
Subject: [PATCH] Add support to read gpu_metrics version 1.2

gpu_metrics version 1.2 provides atomic timestamp. Use this timestamp.

Signed-off-by: Harish Kasiviswanathan <Harish.Kasiviswanathan@amd.com>
Change-Id: I7a1a675f53b93718f34b1f2979173e9064e0ef93


[ROCm/amdsmi commit: 6b10a7761bda137c429682d07c162156e40f9b26]
---
 projects/amdsmi/include/rocm_smi/rocm_smi.h |  1 +
 projects/amdsmi/src/rocm_smi_gpu_metrics.cc | 49 ++++++++++++++++-----
 2 files changed, 40 insertions(+), 10 deletions(-)

diff --git a/projects/amdsmi/include/rocm_smi/rocm_smi.h b/projects/amdsmi/include/rocm_smi/rocm_smi.h
index f19648d347..aecc4c776e 100755
--- a/projects/amdsmi/include/rocm_smi/rocm_smi.h
+++ b/projects/amdsmi/include/rocm_smi/rocm_smi.h
@@ -834,6 +834,7 @@ struct metrics_table_header_t {
 // The content version increments when gpu_metrics is extended with new and/or
 // existing field sizes are changed.
 #define RSMI_GPU_METRICS_API_CONTENT_VER_1 1
+#define RSMI_GPU_METRICS_API_CONTENT_VER_2 2
 
 // This should match NUM_HBM_INSTANCES
 #define RSMI_NUM_HBM_INSTANCES 4
diff --git a/projects/amdsmi/src/rocm_smi_gpu_metrics.cc b/projects/amdsmi/src/rocm_smi_gpu_metrics.cc
index 936802312f..f8e53b4ccb 100755
--- a/projects/amdsmi/src/rocm_smi_gpu_metrics.cc
+++ b/projects/amdsmi/src/rocm_smi_gpu_metrics.cc
@@ -116,13 +116,19 @@ typedef struct {
   uint8_t       pcie_link_speed;  // in 0.1 GT/s
 } rsmi_gpu_metrics_v_1_0_t;
 
+typedef struct {
+  rsmi_gpu_metrics_t base;
+  uint64_t           firmware_timestamp;
+} rsmi_gpu_metrics_v_1_2;
 
 static rsmi_status_t GetGPUMetricsFormat1(uint32_t dv_ind,
                                 rsmi_gpu_metrics_t *data, uint8_t content_v) {
-  assert(content_v != RSMI_GPU_METRICS_API_CONTENT_VER_1);
-  if (content_v == RSMI_GPU_METRICS_API_CONTENT_VER_1) {
+  assert(content_v != RSMI_GPU_METRICS_API_CONTENT_VER_1 &&
+         content_v != RSMI_GPU_METRICS_API_CONTENT_VER_2 );
+  if (content_v == RSMI_GPU_METRICS_API_CONTENT_VER_1 ||
+      content_v == RSMI_GPU_METRICS_API_CONTENT_VER_2 ) {
     // This function shouldn't be called if content version is
-    // RSMI_GPU_METRICS_API_CONTENT_VER_1.
+    // RSMI_GPU_METRICS_API_CONTENT_VER_1 or RSMI_GPU_METRICS_API_CONTENT_VER_2
     return RSMI_STATUS_INVALID_ARGS;
   }
   void *metric_data = nullptr;
@@ -204,11 +210,29 @@ static rsmi_status_t GetGPUMetricsFormat1(uint32_t dv_ind,
   return RSMI_STATUS_SUCCESS;
 }
 
+// Translate gpu_metrics version 1.2 to rsmi_gpu_metrics_t. gpu_metrics
+// version 1.2 provides timestamp provided by the firmware. This timestamp
+// is sampled atomically along when gpu_metric information. Use this
+// timestamp instead of system_clock_counter
+
+static void map_gpu_metrics_1_2_to_rsmi_gpu_metrics_t(
+        const rsmi_gpu_metrics_v_1_2 *gpu_metrics_v_1_2,
+        rsmi_gpu_metrics_t *rsmi_gpu_metrics)
+{
+    memcpy(rsmi_gpu_metrics, &gpu_metrics_v_1_2->base,
+           sizeof(rsmi_gpu_metrics_t));
+    // firmware_timestamp is at 10ns resolution
+    rsmi_gpu_metrics->system_clock_counter =
+          gpu_metrics_v_1_2->firmware_timestamp * 10;
+}
+
+
 rsmi_status_t
 rsmi_dev_gpu_metrics_info_get(uint32_t dv_ind, rsmi_gpu_metrics_t *smu) {
   TRY
   DEVICE_MUTEX
   CHK_SUPPORT_NAME_ONLY(smu)
+  rsmi_gpu_metrics_v_1_2 smu_v_1_2;
   rsmi_status_t ret;
 
   if (!dev->gpu_metrics_ver().structure_size) {
@@ -222,15 +246,20 @@ rsmi_dev_gpu_metrics_info_get(uint32_t dv_ind, rsmi_gpu_metrics_t *smu) {
   // only supports gpu_metrics_v1_x version
   if (dev->gpu_metrics_ver().format_revision != 1) {
     return RSMI_STATUS_NOT_SUPPORTED;
-  } else {  // format_revision == 1
-    if (dev->gpu_metrics_ver().content_revision ==
-                                           RSMI_GPU_METRICS_API_CONTENT_VER_1) {
-      ret = GetDevBinaryBlob(amd::smi::kDevGpuMetrics, dv_ind,
+  }
+
+  if (dev->gpu_metrics_ver().content_revision ==
+                                        RSMI_GPU_METRICS_API_CONTENT_VER_1) {
+        ret = GetDevBinaryBlob(amd::smi::kDevGpuMetrics, dv_ind,
                                            sizeof(rsmi_gpu_metrics_t), smu);
-    } else {
-      ret = GetGPUMetricsFormat1(dv_ind, smu,
+  } else if (dev->gpu_metrics_ver().content_revision ==
+                                        RSMI_GPU_METRICS_API_CONTENT_VER_2) {
+        ret = GetDevBinaryBlob(amd::smi::kDevGpuMetrics, dv_ind,
+                                sizeof(rsmi_gpu_metrics_v_1_2), &smu_v_1_2);
+        map_gpu_metrics_1_2_to_rsmi_gpu_metrics_t(&smu_v_1_2, smu);
+  } else {
+            ret = GetGPUMetricsFormat1(dv_ind, smu,
                                      dev->gpu_metrics_ver().content_revision);
-    }
   }
 
   if (ret != RSMI_STATUS_SUCCESS) {