Add 'projects/rdc/' from commit '5ae7eeb3550d4cb14cbc31d3022e545b054f1ad1'

git-subtree-dir: projects/rdc
git-subtree-mainline: a68afa42a1
git-subtree-split: 5ae7eeb355

projects/rdc/python_binding/README.md

@@ -0,0 +1,36 @@
+# Quick start
+If you do not have the RDC installed, please specify the RDC library path using:
+
+    $ export LD_LIBRARY_PATH=<rdc_libs_path>
+
+Then you can run RdcReader in python_binding folder:
+
+    $ python RdcReader.py
+
+# Prometheus plugin
+Install the prometheus_client:
+
+    $ pip install prometheus_client
+
+Start the rdcd with auth and then run plugin to connect to it:
+
+    $ python rdc_prometheus.py
+
+Check the options of the plugin:
+
+    $ python rdc_prometheus.py --help
+
+Verify the plugin is running:
+
+    $ curl localhost:5000
+
+In the managment computer, install the Prometheus from
+https://github.com/prometheus/prometheus
+
+Modify the file prometheus_targets.json to add the compute nodes running the plugin.
+Start the Prometheus
+
+    $ prometheus --config.file=<full path of the rdc_prometheus_example.yml>
+
+Browse to localhost:9090 in the management computer for metrics from RDC.
+

projects/rdc/python_binding/README_rdc_rest_api.txt

@@ -0,0 +1,102 @@
+# RDC REST API
+
+## Overview
+This REST API provides functionalities to:
+- Discover available GPUs on a node.
+- Configure and manage GPU monitoring queries.
+- Retrieve GPU metrics based on configured queries.
+
+The API is built using Flask and interacts with the RDC library to monitor GPU usage and performance metrics.
+
+## Installation
+### Prerequisites
+- Python 3.x
+- Flask
+- RDC Library (`librdc_bootstrap.so` must be available and accessible)
+
+### Install Dependencies
+```sh
+pip install flask
+```
+
+## Running the API
+1. Ensure `librdc_bootstrap.so` is in the library path:
+   ```sh
+   export LD_LIBRARY_PATH=/path/to/librdc_bootstrap.so:$LD_LIBRARY_PATH
+   ```
+2. Run the API:
+   ```sh
+   python rdc_rest_api.py
+   ```
+
+The API will start and listen on `http://0.0.0.0:50052`.
+
+## API Endpoints
+
+### 1. Discover GPUs
+**GET** `/rdc/discovery`
+#### Response:
+```json
+{
+  "0": "GPU Name",
+  "1": "GPU Name"
+}
+```
+
+### 2. Create Query Criteria
+**POST** `/rdc/query_criteria`
+#### Request Body:
+```json
+{
+  "gpu_index": [0,1],
+  "metrics": ["RDC_FI_GPU_CLOCK", "RDC_FI_GPU_TEMP"]
+}
+```
+#### Response:
+```json
+{
+  "query_id": "G-1-F-2"
+}
+```
+
+### 3. Get Query Criteria
+**GET** `/rdc/query_criteria/<query_id>`
+#### Response:
+```json
+{
+  "gpu_index": [0,1],
+  "metrics": ["RDC_FI_GPU_CLOCK", "RDC_FI_GPU_TEMP"],
+  "query_id": "G-1-F-2"
+}
+```
+
+### 4. Delete Query Criteria
+**DELETE** `/rdc/query_criteria/<query_id>`
+#### Response:
+```json
+{
+  "message": "Deleted successfully"
+}
+```
+
+### 5. Retrieve GPU Metrics
+**GET** `/rdc/gpu_metrics/<query_id>`
+#### Response:
+```json
+[
+  {
+    "gpu_index": 0,
+    "RDC_FI_GPU_CLOCK": 1450,
+    "RDC_FI_GPU_TEMP": 32
+  },
+  {
+    "gpu_index": 1,
+    "RDC_FI_GPU_CLOCK": 736,
+    "RDC_FI_GPU_TEMP": 35
+  }
+]
+```
+
+## Notes
+- Ensure `librdc_bootstrap.so` is properly linked.
+- The API should be run on a system with RDC installed and GPUs accessible.

projects/rdc/python_binding/RdcReader.py

@@ -0,0 +1,179 @@
+import os,time
+from rdc_bootstrap import *
+from RdcUtil import RdcUtil
+from typing import Dict
+
+default_field_ids = [
+        rdc_field_t.RDC_FI_GPU_MEMORY_USAGE,
+        rdc_field_t.RDC_FI_GPU_MEMORY_TOTAL,
+        rdc_field_t.RDC_FI_GPU_MM_ENC_UTIL,
+        rdc_field_t.RDC_FI_GPU_MM_DEC_UTIL,
+        rdc_field_t.RDC_FI_GPU_MEMORY_ACTIVITY,
+        rdc_field_t.RDC_FI_GPU_MEMORY_MAX_BANDWIDTH,
+        rdc_field_t.RDC_FI_GPU_MEMORY_CUR_BANDWIDTH,
+        rdc_field_t.RDC_FI_OAM_ID,
+        rdc_field_t.RDC_FI_POWER_USAGE,
+        rdc_field_t.RDC_FI_GPU_CLOCK,
+        rdc_field_t.RDC_FI_GPU_UTIL,
+        rdc_field_t.RDC_FI_GPU_TEMP,
+        rdc_field_t.RDC_FI_GPU_MEMORY_USAGE
+]
+
+default_unit_coverter = {
+        rdc_field_t.RDC_FI_GPU_MEMORY_USAGE: 0.000001, # MegaBytes
+        rdc_field_t.RDC_FI_GPU_MEMORY_TOTAL: 0.000001, # MegaBytes
+        rdc_field_t.RDC_FI_POWER_USAGE: 0.000001, # Watts
+        rdc_field_t.RDC_FI_GPU_CLOCK: 0.000001, # MHz
+        rdc_field_t.RDC_FI_GPU_TEMP: 0.001, # degree
+}
+
+class RdcReader:
+    # To run the RDC in embedded mode, set the ip_port = None
+    def __init__(self, ip_port = "localhost:50051", field_ids = default_field_ids,
+            unit_converter: Dict[int, float] = default_unit_coverter,
+            update_freq = 10000000, max_keep_age = 3600.0 , max_keep_samples = 1000,
+            field_group_name = "rdc_reader_field_group", gpu_group_name = "rdc_reader_gpu_group",
+            gpu_indexes = None, root_ca = "/etc/rdc/client/certs/rdc_cacert.pem",
+            client_cert = "/etc/rdc/client/certs/rdc_client_cert.pem",
+            client_key = "/etc/rdc/client/private/rdc_client_cert.key"):
+        result = rdc.rdc_init(0)
+        if rdc_status_t(result) != rdc_status_t.RDC_ST_OK:
+            raise Exception("RdcReader init fail: " + str(result))
+
+        self.rdc_util = RdcUtil()
+
+        self.unit_converter = unit_converter
+        self.rdc_handle = c_void_p()
+
+        self.is_standalone = True
+        if not ip_port:  # embedded
+            self.is_standalone = False
+            result = rdc.rdc_start_embedded(rdc_operation_mode_t.RDC_OPERATION_MODE_AUTO, self.rdc_handle)
+            if rdc_status_t(result) != rdc_status_t.RDC_ST_OK:
+                raise Exception("RdcReader start as embedded fail: " + str(result))
+        else: # standalone
+            if root_ca == None or client_cert == None or client_key == None:
+                with_auth = False
+                root_ca_str = client_cert_str = client_key_str = None
+            else:
+                with_auth = True
+                root_ca_str = self.rdc_util.read_file(root_ca)
+                client_cert_str = self.rdc_util.read_file(client_cert)
+                client_key_str = self.rdc_util.read_file(client_key)
+
+            result = rdc.rdc_connect(ip_port.encode('utf-8'), self.rdc_handle, root_ca_str, client_cert_str, client_key_str)
+            if rdc_status_t(result) != rdc_status_t.RDC_ST_OK:
+                raise Exception("RdcReader standalone auth(" + str(with_auth) + ") connect to " + ip_port+ " fail: " + str(result))
+
+        # Create the GPU group
+        self.gpu_group_name = gpu_group_name.encode()
+        if gpu_indexes == None:
+            self.gpu_indexes = self.rdc_util.get_all_gpu_indexes(self.rdc_handle)
+        else:
+            self.gpu_indexes = []
+            for idx in gpu_indexes:
+                idx_str = str(idx)
+                encoded = idx_str.encode("utf-8")
+                phys_gpu = ctypes.c_uint32()
+                part_idx = ctypes.c_uint32()
+                if rdc.rdc_is_partition_string(encoded):
+                    rc = rdc.rdc_parse_partition_string(encoded, ctypes.byref(phys_gpu), ctypes.byref(part_idx))
+                    if not rc:
+                        raise Exception("Rdc failed to parse partition string")
+                    info = rdc_entity_info_t()
+                    info.device_type   = 0 #RDC_DEVICE_TYPE_GPU
+                    info.entity_role   = 1 #RDC_DEVICE_ROLE_PARTITION
+                    info.instance_index = part_idx
+                    info.device_index   = phys_gpu
+                    entity = rdc.rdc_get_entity_index_from_info(info)
+                    self.gpu_indexes.append(entity)
+                else:
+                    self.gpu_indexes.append(int(idx_str))
+        self.gpu_group_id, gpu_group_created = self.rdc_util.create_gpu_group(self.rdc_handle, self.gpu_group_name, self.gpu_indexes)
+
+        # Create the field group
+        self.field_ids = field_ids
+        self.field_group_name = field_group_name.encode()
+        self.field_group_id, field_group_created = self.rdc_util.create_field_group(self.rdc_handle, self.field_group_name, self.field_ids)
+
+        # Watch the fields
+        self.update_freq = update_freq
+        self.max_keep_age = max_keep_age
+        self.max_keep_samples = max_keep_samples
+
+        # Unwatch first to clean up what left from last run
+        rdc.rdc_field_unwatch(self.rdc_handle, self.gpu_group_id, self.field_group_id)
+        result = rdc.rdc_field_watch(self.rdc_handle, self.gpu_group_id,
+                 self.field_group_id, self.update_freq, self.max_keep_age, self.max_keep_samples);
+
+        if rdc_status_t(result) != rdc_status_t.RDC_ST_OK:
+               raise Exception("RdcReader fail to watch group " + str(self.gpu_group_id) + ", field group " + str(self.field_group_id) + ":" + str(result))
+
+    # Process the fields periodically
+    def process(self):
+        has_succeed = False
+        for gindex in self.gpu_indexes:
+            for fid in self.field_ids:
+                value = rdc_field_value()
+                result = rdc.rdc_field_get_latest_value(self.rdc_handle,
+                        gindex, fid, value)
+
+                if rdc_status_t(result) == rdc_status_t.RDC_ST_OK:
+                    # Convert the unit
+                    if self.unit_converter != None and fid in self.unit_converter:
+                        if value.type.value == rdc_field_type_t.INTEGER:
+                            value.value.l_int = int(value.value.l_int * self.unit_converter[fid])
+                        if value.type.value == rdc_field_type_t.DOUBLE:
+                            value.value.dbl = int(value.value.dbl * self.unit_converter[fid])
+                    # convert from double to l_int
+                    if value.type.value == rdc_field_type_t.DOUBLE:
+                        value.value.l_int = int(value.value.dbl)
+                    self.handle_field(gindex, value)
+                    has_succeed = True
+
+        self.process_other_fields()
+
+        if len(self.gpu_indexes) != 0 and len(self.field_ids) != 0 and has_succeed == False:
+            self.try_reconnect()
+
+    def process_other_fields(self):
+        pass
+
+    def try_reconnect(self):
+        if self.is_standalone == False:
+            return
+
+        try:
+            # When rdcd restart, the GPU and field group need to be re-created.
+            self.gpu_group_id, gpu_group_created = self.rdc_util.create_gpu_group(self.rdc_handle, self.gpu_group_name, self.gpu_indexes)
+            self.field_group_id, field_group_created = self.rdc_util.create_field_group(self.rdc_handle, self.field_group_name, self.field_ids)
+
+            # rdcd restart requires to watch the group again
+            if gpu_group_created or field_group_created:
+                result = rdc.rdc_field_watch(self.rdc_handle, self.gpu_group_id,
+                        self.field_group_id, self.update_freq, self.max_keep_age, self.max_keep_samples);
+                if rdc_status_t(result) != rdc_status_t.RDC_ST_OK:
+                     raise Exception("RdcReader fail to watch group " + str(self.gpu_group_id) + ", field group " + str(self.field_group_id) + ":" + str(result))
+        except Exception as e:
+            print(e)
+
+
+    def handle_field(self, gpu_index, value):
+
+        info = rdc.rdc_get_info_from_entity_index(gpu_index)
+
+        if info.entity_role == 1: #RDC_DEVICE_ROLE_PARTITION_INSTANCE
+            gpu_str = f"g{info.device_index}.{info.instance_index}"
+        else:
+            gpu_str = str(info.device_index)
+
+        field_name = self.rdc_util.field_id_string(value.field_id)
+        print("%d %s:%d %s:%d" % (value.ts, gpu_str, value.field_id.value, field_name, value.value.l_int))
+
+
+if __name__ == '__main__':
+    # Run the reader in embedded mode
+    reader = RdcReader(ip_port=None, update_freq=1000000)
+    while True:
+        time.sleep(1)
+        reader.process()

projects/rdc/python_binding/RdcUtil.py

@@ -0,0 +1,112 @@
+from rdc_bootstrap import *
+
+class RdcUtil:
+    def __init__(self):
+        pass
+
+    def get_all_gpu_indexes(self, rdc_handle):
+        gpu_count = c_uint32()
+        gpu_index_list = (c_uint32 * RDC_MAX_NUM_DEVICES)()
+
+        result = rdc.rdc_device_get_all(rdc_handle, gpu_index_list, gpu_count)
+        if rdc_status_t(result) != rdc_status_t.RDC_ST_OK:
+            raise Exception("Fail to get all GPus")
+        gpu_indexes = []
+        for index in range(gpu_count.value):
+            gpu_indexes.append(gpu_index_list[index])
+        return gpu_indexes
+
+    def get_all_gpu_groups(self, rdc_handle):
+        all_groups = {}
+        group_count = c_uint32()
+        gpu_group_list = (c_uint32 * RDC_MAX_NUM_GROUPS)()
+
+        result = rdc.rdc_group_get_all_ids(rdc_handle, gpu_group_list, group_count)
+        if rdc_status_t(result) != rdc_status_t.RDC_ST_OK:
+            raise Exception("Fail to get all groups")
+        for index in range(group_count.value):
+            group_id = gpu_group_list[index]
+            group_info = rdc_group_info_t()
+            result = rdc.rdc_group_gpu_get_info(rdc_handle, group_id, group_info)
+            all_groups[group_id] = group_info
+
+        return all_groups
+
+
+    # Create gpu group if not exists
+    # Return <gpu_group_id, is_created>
+    def create_gpu_group(self, rdc_handle, gpu_group_name, gpu_indexes):
+        # Can we reuse the exists one?
+        all_groups = self.get_all_gpu_groups(rdc_handle)
+        for id,group_info in all_groups.items():
+            group_name = group_info.group_name.decode('utf-8')
+            list_gpu_indexes = list(group_info.entity_ids[:group_info.count])
+            if group_name == gpu_group_name:
+                # Reuse existing group
+                if list_gpu_indexes == gpu_indexes:
+                    return id, False
+                else: # delete old group
+                    result = rdc.rdc_group_gpu_destroy(rdc_handle, id)
+                    if rdc_status_t(result) != rdc_status_t.RDC_ST_OK:
+                        raise Exception("Fail to delete the GPU group")
+
+        #Create new gpu group
+        gpu_group_id = c_uint32()
+        result = rdc.rdc_group_gpu_create(rdc_handle, rdc_group_type_t.RDC_GROUP_EMPTY, gpu_group_name, gpu_group_id)
+        if rdc_status_t(result) != rdc_status_t.RDC_ST_OK:
+            raise Exception("Fail to create the GPU group " + group_name)
+
+        #Add GPU index to the group
+        for gpu in gpu_indexes:
+            result = rdc.rdc_group_gpu_add(rdc_handle, gpu_group_id, gpu)
+            if rdc_status_t(result) != rdc_status_t.RDC_ST_OK:
+                raise Exception("Fail to add GPU index " + str(gpu) + " to group " + str(gpu_group_id))
+
+        return gpu_group_id, True
+
+    def create_field_group(self, rdc_handle, field_group_name, field_ids):
+        # Do we need to recreate the field group?
+        field_group_id_list = (rdc_field_grp_t * RDC_MAX_FIELD_IDS_PER_FIELD_GROUP)()
+        field_group_count = c_uint32()
+        result = rdc.rdc_group_field_get_all_ids(rdc_handle, field_group_id_list, field_group_count)
+        if rdc_status_t(result) != rdc_status_t.RDC_ST_OK:
+            raise Exception("Fail to get all field group")
+        for index in range(field_group_count.value):
+            group_info = rdc_field_group_info_t()
+            result = rdc.rdc_group_field_get_info(rdc_handle, field_group_id_list[index], pointer(group_info))
+            if rdc_status_t(result) != rdc_status_t.RDC_ST_OK:
+                raise Exception("Fail to get field group " + str(field_group_id_list[index]) + " info")
+            if group_info.group_name.decode("utf-8") == field_group_name:
+                field_ids_ori = [ e.value for  e in group_info.field_ids[:group_info.count] ]
+                # reuse the old field group
+                if (field_ids == field_ids_ori):
+                    return field_group_id_list[index], False
+                else:
+                    result = rdc.rdc_group_field_destroy(rdc_handle, field_group_id_list[index])
+                    if rdc_status_t(result) != rdc_status_t.RDC_ST_OK:
+                        raise Exception("Fail to delete field group " + str(field_group_id_list[index]))
+
+        #Create new field group
+        fields_c_ids = []
+        for f in field_ids:
+            fields_c_ids.append(rdc_field_t(f))
+        c_ids = ( rdc_field_t * len(field_ids))(*fields_c_ids)
+
+        field_group_id = c_uint32()
+        result = rdc.rdc_group_field_create(rdc_handle, len(field_ids), c_ids, field_group_name, field_group_id)
+        if rdc_status_t(result) != rdc_status_t.RDC_ST_OK:
+            raise Exception("Fail to create field group " + field_group_name.decode("utf-8") +": " + str(result))
+
+        return field_group_id, True
+
+    def field_id_string(self, field_id):
+        return rdc.field_id_string(field_id).decode("utf-8")
+
+    def read_file(self, file_name):
+        try:
+            with open(file_name, 'r') as file:
+              return file.read().encode('utf-8')
+        except Exception as e:
+            print("Fail to read " + file_name + ":" + str(e))
+
+        return None

projects/rdc/python_binding/prometheus_targets.json

@@ -0,0 +1,7 @@
+[
+  {
+    "targets": [
+      "localhost:5000"
+    ]
+  }
+]

projects/rdc/python_binding/rdc_bootstrap.py

@@ -0,0 +1,456 @@
+import os, time
+import ctypes.util
+from ctypes import *
+from enum import Enum
+librdc = "librdc_bootstrap.so"
+
+
+# The python ctypes wrapper for "librdc_bootstrap.so"
+
+# Search librdc_bootstrap.so paths
+current_folder = os.path.dirname(os.path.realpath(__file__))
+rdc_paths = [ "", # without path
+        current_folder+"/../../../lib/",   # package installation
+        current_folder+"/../../../lib64/", # package installation
+        current_folder+"/../build/rdc_libs/" # build from source code
+]
+
+rdc = None
+for r in rdc_paths:
+    try:
+        rdc = CDLL(r+librdc)
+        break
+    except:
+        pass
+
+if rdc == None:
+    print("Unable to load the librdc_bootstrap.so. Set LD_LIBRARY_PATH to the folder containing librdc_bootstrap.so.")
+    exit(1)
+
+
+GPU_ID_INVALID = -1
+RDC_GROUP_ALL_GPUS = -1000
+RDC_JOB_STATS_FIELDS = -1000
+RDC_MAX_STR_LENGTH = 256
+RDC_GROUP_MAX_ENTITIES = 64
+RDC_MAX_NUM_DEVICES = 128
+RDC_MAX_FIELD_IDS_PER_FIELD_GROUP = 128
+RDC_MAX_NUM_GROUPS = 64
+RDC_MAX_NUM_FIELD_GROUPS = 64
+class rdc_status_t(Enum):
+     def from_param(cls, obj):
+          return int(obj)
+     RDC_ST_OK = 0
+     RDC_ST_NOT_SUPPORTED = 1
+     RDC_ST_SMI_ERROR = 2
+     RDC_ST_FAIL_LOAD_MODULE = 3
+     RDC_ST_INVALID_HANDLER = 4
+     RDC_ST_BAD_PARAMETER = 5
+     RDC_ST_NOT_FOUND = 6
+     RDC_ST_CONFLICT = 7
+     RDC_ST_CLIENT_ERROR = 8
+     RDC_ST_ALREADY_EXIST = 9
+     RDC_ST_MAX_LIMIT = 10
+     RDC_ST_INSUFF_RESOURCES = 11
+     RDC_ST_FILE_ERROR = 12
+     RDC_ST_NO_DATA = 13
+     RDC_ST_PERM_ERROR = 14
+     RDC_ST_CORRUPTED_EEPROM = 15
+     RDC_ST_UNKNOWN_ERROR = 4294967295
+
+class rdc_operation_mode_t(c_int):
+     RDC_OPERATION_MODE_AUTO = 0
+     RDC_OPERATION_MODE_MANUAL = 1
+
+class rdc_group_type_t(c_int):
+     RDC_GROUP_DEFAULT = 0
+     RDC_GROUP_EMPTY = 1
+
+class rdc_field_type_t(c_int):
+     INTEGER = 0
+     DOUBLE = 1
+     STRING = 2
+     BLOB = 3
+
+class rdc_metric_type_t(c_int):
+     INVALID = 0
+     GAUGE = 1
+     COUNTER = 2
+     LABEL = 3
+
+class rdc_field_t(c_int):
+
+     RDC_FI_INVALID = 0
+     RDC_FI_GPU_COUNT = 1
+     RDC_FI_DEV_NAME = 2
+     RDC_FI_OAM_ID = 3
+     RDC_FI_GPU_CLOCK = 100
+     RDC_FI_MEM_CLOCK = 101
+     RDC_FI_MEMORY_TEMP = 200
+     RDC_FI_GPU_TEMP = 201
+     RDC_FI_POWER_USAGE = 300
+     RDC_FI_PCIE_TX = 400
+     RDC_FI_PCIE_RX = 401
+     RDC_FI_PCIE_BANDWIDTH = 402
+     RDC_FI_GPU_UTIL = 500
+     RDC_FI_GPU_MEMORY_USAGE = 501
+     RDC_FI_GPU_MEMORY_TOTAL = 502
+     RDC_FI_GPU_MM_ENC_UTIL = 503
+     RDC_FI_GPU_MM_DEC_UTIL = 504
+     RDC_FI_GPU_MEMORY_ACTIVITY = 505
+     RDC_FI_GPU_MEMORY_MAX_BANDWIDTH = 506
+     RDC_FI_GPU_MEMORY_CUR_BANDWIDTH = 507
+     RDC_FI_GPU_BUSY_PERCENT = 508
+     RDC_FI_GPU_PAGE_RETRIED = 550
+     RDC_FI_ECC_CORRECT_TOTAL = 600
+     RDC_FI_ECC_UNCORRECT_TOTAL = 601
+     RDC_FI_ECC_SDMA_CE = 602
+     RDC_FI_ECC_SDMA_UE = 603
+     RDC_FI_ECC_GFX_CE = 604
+     RDC_FI_ECC_GFX_UE = 605
+     RDC_FI_ECC_MMHUB_CE = 606
+     RDC_FI_ECC_MMHUB_UE = 607
+     RDC_FI_ECC_ATHUB_CE = 608
+     RDC_FI_ECC_ATHUB_UE = 609
+     RDC_FI_ECC_PCIE_BIF_CE = 610
+     RDC_FI_ECC_PCIE_BIF_UE = 611
+     RDC_FI_ECC_HDP_CE = 612
+     RDC_FI_ECC_HDP_UE = 613
+     RDC_FI_ECC_XGMI_WAFL_CE = 614
+     RDC_FI_ECC_XGMI_WAFL_UE = 615
+     RDC_FI_ECC_DF_CE = 616
+     RDC_FI_ECC_DF_UE = 617
+     RDC_FI_ECC_SMN_CE = 618
+     RDC_FI_ECC_SMN_UE = 619
+     RDC_FI_ECC_SEM_CE = 620
+     RDC_FI_ECC_SEM_UE = 621
+     RDC_FI_ECC_MP0_CE = 622
+     RDC_FI_ECC_MP0_UE = 623
+     RDC_FI_ECC_MP1_CE = 624
+     RDC_FI_ECC_MP1_UE = 625
+     RDC_FI_ECC_FUSE_CE = 626
+     RDC_FI_ECC_FUSE_UE = 627
+     RDC_FI_ECC_UMC_CE = 628
+     RDC_FI_ECC_UMC_UE = 629
+     RDC_FI_ECC_MCA_CE = 630
+     RDC_FI_ECC_MCA_UE = 631
+     RDC_FI_ECC_VCN_CE = 632
+     RDC_FI_ECC_VCN_UE = 633
+     RDC_FI_ECC_JPEG_CE = 634
+     RDC_FI_ECC_JPEG_UE = 635
+     RDC_FI_ECC_IH_CE = 636
+     RDC_FI_ECC_IH_UE = 637
+     RDC_FI_ECC_MPIO_CE = 638
+     RDC_FI_ECC_MPIO_UE = 639
+     RDC_FI_XGMI_0_READ_KB = 700
+     RDC_FI_XGMI_1_READ_KB = 701
+     RDC_FI_XGMI_2_READ_KB = 702
+     RDC_FI_XGMI_3_READ_KB = 703
+     RDC_FI_XGMI_4_READ_KB = 704
+     RDC_FI_XGMI_5_READ_KB = 705
+     RDC_FI_XGMI_6_READ_KB = 706
+     RDC_FI_XGMI_7_READ_KB = 707
+     RDC_FI_XGMI_0_WRITE_KB = 708
+     RDC_FI_XGMI_1_WRITE_KB = 709
+     RDC_FI_XGMI_2_WRITE_KB = 710
+     RDC_FI_XGMI_3_WRITE_KB = 711
+     RDC_FI_XGMI_4_WRITE_KB = 712
+     RDC_FI_XGMI_5_WRITE_KB = 713
+     RDC_FI_XGMI_6_WRITE_KB = 714
+     RDC_FI_XGMI_7_WRITE_KB = 715
+     RDC_FI_XGMI_TOTAL_READ_KB = 716
+     RDC_FI_XGMI_TOTAL_WRITE_KB = 717
+     RDC_FI_PROF_OCCUPANCY_PERCENT = 800
+     RDC_FI_PROF_ACTIVE_CYCLES = 801
+     RDC_FI_PROF_ACTIVE_WAVES = 802
+     RDC_FI_PROF_ELAPSED_CYCLES = 803
+     RDC_FI_PROF_TENSOR_ACTIVE_PERCENT = 804
+     RDC_FI_PROF_GPU_UTIL_PERCENT = 805
+     RDC_FI_PROF_EVAL_MEM_R_BW = 806
+     RDC_FI_PROF_EVAL_MEM_W_BW = 807
+     RDC_FI_PROF_EVAL_FLOPS_16 = 808
+     RDC_FI_PROF_EVAL_FLOPS_32 = 809
+     RDC_FI_PROF_EVAL_FLOPS_64 = 810
+     RDC_FI_PROF_VALU_PIPE_ISSUE_UTIL = 811
+     RDC_FI_PROF_SM_ACTIVE = 812
+     RDC_FI_PROF_OCC_PER_ACTIVE_CU = 813
+     RDC_FI_PROF_OCC_ELAPSED = 814
+     RDC_FI_PROF_EVAL_FLOPS_16_PERCENT = 815
+     RDC_FI_PROF_EVAL_FLOPS_32_PERCENT = 816
+     RDC_FI_PROF_EVAL_FLOPS_64_PERCENT = 817
+     RDC_FI_PROF_CPC_CPC_STAT_BUSY = 818
+     RDC_FI_PROF_CPC_CPC_STAT_IDLE = 819
+     RDC_FI_PROF_CPC_CPC_STAT_STALL = 820
+     RDC_FI_PROF_CPC_CPC_TCIU_BUSY = 821
+     RDC_FI_PROF_CPC_CPC_TCIU_IDLE = 822
+     RDC_FI_PROF_CPC_CPC_UTCL2IU_BUSY = 823
+     RDC_FI_PROF_CPC_CPC_UTCL2IU_IDLE = 824
+     RDC_FI_PROF_CPC_CPC_UTCL2IU_STALL = 825
+     RDC_FI_PROF_CPC_ME1_BUSY_FOR_PACKET_DECODE = 826
+     RDC_FI_PROF_CPC_ME1_DC0_SPI_BUSY = 827
+     RDC_FI_PROF_CPC_UTCL1_STALL_ON_TRANSLATION = 828
+     RDC_FI_PROF_CPC_ALWAYS_COUNT = 829
+     RDC_FI_PROF_CPC_ADC_VALID_CHUNK_NOT_AVAIL = 830
+     RDC_FI_PROF_CPC_ADC_DISPATCH_ALLOC_DONE = 831
+     RDC_FI_PROF_CPC_ADC_VALID_CHUNK_END = 832
+     RDC_FI_PROF_CPC_SYNC_FIFO_FULL_LEVEL = 833
+     RDC_FI_PROF_CPC_SYNC_FIFO_FULL = 834
+     RDC_FI_PROF_CPC_GD_BUSY = 835
+     RDC_FI_PROF_CPC_TG_SEND = 836
+     RDC_FI_PROF_CPC_WALK_NEXT_CHUNK = 837
+     RDC_FI_PROF_CPC_STALLED_BY_SE0_SPI = 838
+     RDC_FI_PROF_CPC_STALLED_BY_SE1_SPI = 839
+     RDC_FI_PROF_CPC_STALLED_BY_SE2_SPI = 840
+     RDC_FI_PROF_CPC_STALLED_BY_SE3_SPI = 841
+     RDC_FI_PROF_CPC_LTE_ALL = 842
+     RDC_FI_PROF_CPC_SYNC_WRREQ_FIFO_BUSY = 843
+     RDC_FI_PROF_CPC_CANE_BUSY = 844
+     RDC_FI_PROF_CPC_CANE_STALL = 845
+     RDC_FI_PROF_CPF_CMP_UTCL1_STALL_ON_TRANSLATION = 846
+     RDC_FI_PROF_CPF_CPF_STAT_BUSY = 847
+     RDC_FI_PROF_CPF_CPF_STAT_IDLE = 848
+     RDC_FI_PROF_CPF_CPF_STAT_STALL = 849
+     RDC_FI_PROF_CPF_CPF_TCIU_BUSY = 850
+     RDC_FI_PROF_CPF_CPF_TCIU_IDLE = 851
+     RDC_FI_PROF_CPF_CPF_TCIU_STALL = 852
+     RDC_FI_PROF_SIMD_UTILIZATION = 853
+     RDC_EVNT_XGMI_0_NOP_TX = 1000
+     RDC_EVNT_XGMI_0_REQ_TX = 1001
+     RDC_EVNT_XGMI_0_RESP_TX = 1002
+     RDC_EVNT_XGMI_0_BEATS_TX = 1003
+     RDC_EVNT_XGMI_1_NOP_TX = 1004
+     RDC_EVNT_XGMI_1_REQ_TX = 1005
+     RDC_EVNT_XGMI_1_RESP_TX = 1006
+     RDC_EVNT_XGMI_1_BEATS_TX = 1007
+     RDC_EVNT_XGMI_0_THRPUT = 1500
+     RDC_EVNT_XGMI_1_THRPUT = 1501
+     RDC_EVNT_XGMI_2_THRPUT = 1502
+     RDC_EVNT_XGMI_3_THRPUT = 1503
+     RDC_EVNT_XGMI_4_THRPUT = 1504
+     RDC_EVNT_XGMI_5_THRPUT = 1505
+     RDC_EVNT_NOTIF_VMFAULT = 2000
+     RDC_EVNT_NOTIF_THERMAL_THROTTLE = 2001
+     RDC_EVNT_NOTIF_PRE_RESET = 2002
+     RDC_EVNT_NOTIF_POST_RESET = 2003
+     RDC_EVNT_NOTIF_MIGRATE_START = 2004
+     RDC_EVNT_NOTIF_MIGRATE_END = 2005
+     RDC_EVNT_NOTIF_PAGE_FAULT_START = 2006
+     RDC_EVNT_NOTIF_PAGE_FAULT_END = 2007
+     RDC_EVNT_NOTIF_QUEUE_EVICTION = 2008
+     RDC_EVNT_NOTIF_QUEUE_RESTORE = 2009
+     RDC_EVNT_NOTIF_UNMAP_FROM_GPU = 2010
+     RDC_EVNT_NOTIF_PROCESS_START = 2011
+     RDC_EVNT_NOTIF_PROCESS_END = 2012
+     RDC_HEALTH_XGMI_ERROR = 3000
+     RDC_HEALTH_PCIE_REPLAY_COUNT = 3001
+     RDC_HEALTH_RETIRED_PAGE_NUM = 3002
+     RDC_HEALTH_PENDING_PAGE_NUM = 3003
+     RDC_HEALTH_RETIRED_PAGE_LIMIT = 3004
+     RDC_HEALTH_EEPROM_CONFIG_VALID = 3005
+     RDC_HEALTH_POWER_THROTTLE_TIME = 3006
+     RDC_HEALTH_THERMAL_THROTTLE_TIME = 3007
+
+     _rdc_metric_type_lookup = {
+        RDC_FI_INVALID: rdc_metric_type_t.INVALID,
+        RDC_FI_GPU_COUNT: rdc_metric_type_t.LABEL,
+        RDC_FI_DEV_NAME: rdc_metric_type_t.LABEL,
+        RDC_FI_OAM_ID: rdc_metric_type_t.LABEL,
+        RDC_FI_GPU_MEMORY_TOTAL: rdc_metric_type_t.COUNTER,
+        RDC_FI_ECC_CORRECT_TOTAL: rdc_metric_type_t.COUNTER,
+        RDC_FI_ECC_UNCORRECT_TOTAL: rdc_metric_type_t.COUNTER,
+        RDC_EVNT_NOTIF_VMFAULT: rdc_metric_type_t.COUNTER,
+        RDC_EVNT_NOTIF_THERMAL_THROTTLE: rdc_metric_type_t.COUNTER,
+        RDC_EVNT_NOTIF_PRE_RESET: rdc_metric_type_t.COUNTER,
+        RDC_EVNT_NOTIF_POST_RESET: rdc_metric_type_t.COUNTER,
+        RDC_EVNT_NOTIF_MIGRATE_START: rdc_metric_type_t.COUNTER,
+        RDC_EVNT_NOTIF_MIGRATE_END: rdc_metric_type_t.COUNTER,
+        RDC_EVNT_NOTIF_PAGE_FAULT_START: rdc_metric_type_t.COUNTER,
+        RDC_EVNT_NOTIF_PAGE_FAULT_END: rdc_metric_type_t.COUNTER,
+        RDC_EVNT_NOTIF_QUEUE_EVICTION: rdc_metric_type_t.COUNTER,
+        RDC_EVNT_NOTIF_QUEUE_RESTORE: rdc_metric_type_t.COUNTER,
+        RDC_EVNT_NOTIF_UNMAP_FROM_GPU: rdc_metric_type_t.COUNTER,
+        RDC_EVNT_NOTIF_PROCESS_START: rdc_metric_type_t.COUNTER,
+        RDC_EVNT_NOTIF_PROCESS_END: rdc_metric_type_t.COUNTER,
+     }
+
+     @classmethod
+     def get_rdc_metric_type(cls, rdc_metric_t):
+        if isinstance(rdc_metric_t, str):
+            rdc_metric_t = getattr(cls, rdc_metric_t, None)
+        
+        # If the metric was found, do the lookup, otherwise default GAUGE
+        if rdc_metric_t is not None:
+            return cls._rdc_metric_type_lookup.get(rdc_metric_t, rdc_metric_type_t.GAUGE)
+        return rdc_metric_type_t.GAUGE
+
+     @classmethod
+     def get_field_name(cls, value):
+        for attr_name, attr_value in cls.__dict__.items():
+            if isinstance(attr_value, int) and attr_value == value:
+                return attr_name
+        return "Unknown field value"    
+
+rdc_handle_t = c_void_p
+rdc_gpu_group_t = c_uint32
+rdc_field_grp_t = c_uint32
+class rdc_device_attributes_t(Structure):
+    _fields_ = [
+            ("device_name", c_char*256)
+            ]
+
+class rdc_group_info_t(Structure):
+    _fields_ = [
+            ("count", c_uint32)
+            ,("group_name", c_char*256)
+            ,("entity_ids", c_uint32*64)
+            ]
+
+class rdc_stats_summary_t(Structure):
+    _fields_ = [
+            ("max_value", c_uint64)
+            ,("min_value", c_uint64)
+            ,("average", c_uint64)
+            ,("standard_deviation", c_double)
+            ]
+
+class rdc_gpu_usage_info_t(Structure):
+    _fields_ = [
+            ("gpu_id", c_uint32)
+            ,("start_time", c_uint64)
+            ,("end_time", c_uint64)
+            ,("energy_consumed", c_uint64)
+            ,("ecc_correct", c_uint64)
+            ,("ecc_uncorrect", c_uint64)
+            ,("pcie_tx", rdc_stats_summary_t)
+            ,("pcie_rx", rdc_stats_summary_t)
+            ,("pcie_total", rdc_stats_summary_t)
+            ,("power_usage", rdc_stats_summary_t)
+            ,("gpu_clock", rdc_stats_summary_t)
+            ,("memory_clock", rdc_stats_summary_t)
+            ,("gpu_utilization", rdc_stats_summary_t)
+            ,("gpu_temperature", rdc_stats_summary_t)
+            ,("max_gpu_memory_used", c_uint64)
+            ,("memory_utilization", rdc_stats_summary_t)
+            ]
+
+class rdc_process_status_info_t(Structure):
+    _fields_ = [
+            ("pid", c_uint32)
+            ,("process_name", c_char*256)
+            ,("start_time", c_uint64)
+            ,("stop_time", c_uint64)
+            ]
+
+class rdc_job_info_t(Structure):
+    _fields_ = [
+            ("num_gpus", c_uint32)
+            ,("summary", rdc_gpu_usage_info_t)
+            ,("gpus", rdc_gpu_usage_info_t*16)
+            ,("num_processes", c_uint32)
+            ,("processes", rdc_process_status_info_t*64)
+            ]
+
+class rdc_anonymous_0(ctypes.Union):
+    _fields_ = [
+            ("l_int", c_int64)
+            ,("dbl", c_double)
+            ,("str", c_char*256)
+            ]
+
+class rdc_field_value(Structure):
+    _fields_ = [
+            ("field_id", rdc_field_t)
+            ,("status", c_int)
+            ,("ts", c_uint64)
+            ,("type", rdc_field_type_t)
+            ,("value", rdc_anonymous_0)
+            ]
+
+class rdc_field_group_info_t(Structure):
+    _fields_ = [
+            ("count", c_uint32)
+            ,("group_name", c_char*256)
+            ,("field_ids", rdc_field_t*128)
+            ]
+
+class rdc_job_group_info_t(Structure):
+    _fields_ = [
+            ("job_id", c_char*256)
+            ,("group_id", rdc_gpu_group_t)
+            ,("start_time", c_uint64)
+            ,("stop_time", c_uint64)
+            ]
+
+class rdc_entity_info_t(Structure):
+    _fields_ = [
+        ("device_index",   c_uint32),
+        ("instance_index", c_uint32),
+        ("entity_role",   c_uint32),
+        ("device_type",   c_uint32),
+    ]
+
+
+rdc.rdc_init.restype = rdc_status_t
+rdc.rdc_init.argtypes = [ c_uint64 ]
+rdc.rdc_shutdown.restype = rdc_status_t
+rdc.rdc_shutdown.argtypes = [  ]
+rdc.rdc_start_embedded.restype = rdc_status_t
+rdc.rdc_start_embedded.argtypes = [ rdc_operation_mode_t,POINTER(rdc_handle_t) ]
+rdc.rdc_stop_embedded.restype = rdc_status_t
+rdc.rdc_stop_embedded.argtypes = [ rdc_handle_t ]
+rdc.rdc_connect.restype = rdc_status_t
+rdc.rdc_connect.argtypes = [ c_char_p,POINTER(rdc_handle_t),c_char_p,c_char_p,c_char_p ]
+rdc.rdc_disconnect.restype = rdc_status_t
+rdc.rdc_disconnect.argtypes = [ rdc_handle_t ]
+rdc.rdc_job_start_stats.restype = rdc_status_t
+rdc.rdc_job_start_stats.argtypes = [ rdc_handle_t,rdc_gpu_group_t,POINTER(c_char),c_uint64 ]
+rdc.rdc_job_get_stats.restype = rdc_status_t
+rdc.rdc_job_get_stats.argtypes = [ rdc_handle_t,POINTER(c_char),POINTER(rdc_job_info_t) ]
+rdc.rdc_job_stop_stats.restype = rdc_status_t
+rdc.rdc_job_stop_stats.argtypes = [ rdc_handle_t,POINTER(c_char) ]
+rdc.rdc_job_remove.restype = rdc_status_t
+rdc.rdc_job_remove.argtypes = [ rdc_handle_t,POINTER(c_char) ]
+rdc.rdc_job_remove_all.restype = rdc_status_t
+rdc.rdc_job_remove_all.argtypes = [ rdc_handle_t ]
+rdc.rdc_field_update_all.restype = rdc_status_t
+rdc.rdc_field_update_all.argtypes = [ rdc_handle_t,c_uint32 ]
+rdc.rdc_device_get_all.restype = rdc_status_t
+rdc.rdc_device_get_all.argtypes = [ rdc_handle_t,POINTER(c_uint32),POINTER(c_uint32) ]
+rdc.rdc_device_get_attributes.restype = rdc_status_t
+rdc.rdc_device_get_attributes.argtypes = [ rdc_handle_t,c_uint32,POINTER(rdc_device_attributes_t) ]
+rdc.rdc_group_gpu_create.restype = rdc_status_t
+rdc.rdc_group_gpu_create.argtypes = [ rdc_handle_t,rdc_group_type_t,c_char_p,POINTER(rdc_gpu_group_t) ]
+rdc.rdc_group_gpu_add.restype = rdc_status_t
+rdc.rdc_group_gpu_add.argtypes = [ rdc_handle_t,rdc_gpu_group_t,c_uint32 ]
+rdc.rdc_group_gpu_get_info.restype = rdc_status_t
+rdc.rdc_group_gpu_get_info.argtypes = [ rdc_handle_t,rdc_gpu_group_t,POINTER(rdc_group_info_t) ]
+rdc.rdc_group_get_all_ids.restype = rdc_status_t
+rdc.rdc_group_get_all_ids.argtypes = [ rdc_handle_t,POINTER(rdc_gpu_group_t),POINTER(c_uint32) ]
+rdc.rdc_group_gpu_destroy.restype = rdc_status_t
+rdc.rdc_group_gpu_destroy.argtypes = [ rdc_handle_t,rdc_gpu_group_t ]
+rdc.rdc_group_field_create.restype = rdc_status_t
+rdc.rdc_group_field_create.argtypes = [ rdc_handle_t,c_uint32,POINTER(rdc_field_t),c_char_p,POINTER(rdc_field_grp_t) ]
+rdc.rdc_group_field_get_info.restype = rdc_status_t
+rdc.rdc_group_field_get_info.argtypes = [ rdc_handle_t,rdc_field_grp_t,POINTER(rdc_field_group_info_t) ]
+rdc.rdc_group_field_get_all_ids.restype = rdc_status_t
+rdc.rdc_group_field_get_all_ids.argtypes = [ rdc_handle_t,POINTER(rdc_field_grp_t),POINTER(c_uint32) ]
+rdc.rdc_group_field_destroy.restype = rdc_status_t
+rdc.rdc_group_field_destroy.argtypes = [ rdc_handle_t,rdc_field_grp_t ]
+rdc.rdc_field_watch.restype = rdc_status_t
+rdc.rdc_field_watch.argtypes = [ rdc_handle_t,rdc_gpu_group_t,rdc_field_grp_t,c_uint64,c_double,c_uint32 ]
+rdc.rdc_field_get_latest_value.restype = rdc_status_t
+rdc.rdc_field_get_latest_value.argtypes = [ rdc_handle_t,c_uint32,rdc_field_t,POINTER(rdc_field_value) ]
+rdc.rdc_field_get_value_since.restype = rdc_status_t
+rdc.rdc_field_get_value_since.argtypes = [ rdc_handle_t,c_uint32,rdc_field_t,c_uint64,POINTER(c_uint64),POINTER(rdc_field_value) ]
+rdc.rdc_field_unwatch.restype = rdc_status_t
+rdc.rdc_field_unwatch.argtypes = [ rdc_handle_t,rdc_gpu_group_t,rdc_field_grp_t ]
+rdc.rdc_status_string.restype = c_char_p
+rdc.rdc_status_string.argtypes = [ rdc_status_t ]
+rdc.field_id_string.restype = c_char_p
+rdc.field_id_string.argtypes = [ rdc_field_t ]
+rdc.get_field_id_from_name.restype = rdc_field_t
+rdc.get_field_id_from_name.argtypes = [ c_char_p ]
+rdc.rdc_get_entity_index_from_info.argtypes = [ rdc_entity_info_t ]
+rdc.rdc_get_entity_index_from_info.restype  = c_uint32
+rdc.rdc_get_info_from_entity_index.argtypes = [c_uint32]
+rdc.rdc_get_info_from_entity_index.restype  = rdc_entity_info_t

projects/rdc/python_binding/rdc_collectd.conf

@@ -0,0 +1,24 @@
+ <Plugin python>
+    ModulePath "/opt/rocm/rdc/python_binding"
+    LogTraces true
+    Interactive false
+    Import "rdc_collectd"
+    <Module rdc_collectd>
+        # Run RDC in embedded mode (default: standalone mode)
+        embedded false
+        # The rdcd IP and port in standalone mode (default: localhost:50051)
+        rdc_ip_port "localhost:50051"
+        # Set this option if the rdcd is running with unauth in standalone mode (default: false)
+        unauth false
+        # The list of fields name needs to be watched (default: fields in the plugin), for example
+        # field_ids "RDC_FI_GPU_TEMP" "RDC_FI_GPU_CLOCK"
+        # The fields update frequency in seconds (default: 10)
+        update_freq 10
+        # The max keep age of the fields in seconds (default: 3600)
+        max_keep_age 3600
+        # The max samples to keep for each field in the cache (default: 1000)
+        max_keep_samples 1000
+        # The list of GPUs to be watched (default: All GPUs), for example
+        # gpu_indexes 0 1
+    </Module>
+ </Plugin>

projects/rdc/python_binding/rdc_collectd.py

@@ -0,0 +1,93 @@
+from RdcReader import RdcReader
+from rdc_bootstrap import *
+import collectd
+
+default_field_ids = [
+        rdc_field_t.RDC_FI_GPU_MEMORY_USAGE,
+        rdc_field_t.RDC_FI_GPU_MEMORY_TOTAL,
+        rdc_field_t.RDC_FI_POWER_USAGE,
+        rdc_field_t.RDC_FI_GPU_CLOCK,
+        rdc_field_t.RDC_FI_GPU_UTIL,
+        rdc_field_t.RDC_FI_GPU_TEMP,
+        rdc_field_t.RDC_FI_GPU_MEMORY_USAGE,
+]
+
+
+class CollectdReader(RdcReader):
+    def __init__(self, rdc_ip_port, field_ids, update_freq, max_keep_age, max_keep_samples,
+            gpu_indexes, rdc_unauth):
+        group_name = "rdc_collectd_plugin_group"
+        field_group_name = "rdc_collectd_plugin_fieldgroup"
+        if rdc_unauth:
+            RdcReader.__init__(self, ip_port = rdc_ip_port, field_ids = field_ids, update_freq=update_freq,
+                    max_keep_age = max_keep_age, max_keep_samples = max_keep_samples,
+                    gpu_indexes = gpu_indexes, field_group_name = field_group_name, gpu_group_name = group_name, root_ca = None)
+        else:
+             RdcReader.__init__(self, ip_port = rdc_ip_port, field_ids = field_ids, update_freq=update_freq,
+                    max_keep_age = max_keep_age, max_keep_samples = max_keep_samples,
+                    gpu_indexes = gpu_indexes, field_group_name = field_group_name, gpu_group_name = group_name)
+
+    def handle_field(self, gpu_index, value):
+        PLUGIN_NAME = "rdc_collectd"
+        field_name = self.rdc_util.field_id_string(value.field_id).lower()
+        collectd.Values(plugin=PLUGIN_NAME,
+                    type_instance= field_name,
+                    type="gauge",
+                    values=[value.value.l_int]).dispatch()
+
+g_reader = None
+
+def config_func(config):
+    global g_reader
+
+    embedded = False # enable embedded if no rdcd
+    rdc_ip_port = "localhost:50051" # rdcd listen address
+    field_ids = default_field_ids # The fields to watch
+    update_freq = 10  # 10 seconds
+    max_keep_age = 3600    # 1 hour
+    max_keep_samples = 1000 # The max samples to keep for each field
+    gpu_indexes = None # All GPus
+    unauth = False  # Enable auth by default
+
+    # Parse configure parameters
+    for node in config.children:
+        key = node.key.lower()
+        if len(node.values) <= 0:
+            print("Missing value in configure " + key)
+            continue
+
+        val = node.values[0]
+        if key == 'embedded' and val == True:
+            embedded = True
+        if key == 'rdc_ip_port':
+            rdc_ip_port = val
+        if key == 'unauth':
+            unauth = val
+        if key == 'field_ids':
+            field_ids = []
+            for f in node.values:
+                field_id = rdc.get_field_id_from_name(str.encode(f))
+                if field_id.value == rdc_field_t.RDC_FI_INVALID:
+                    print("Invalid field '%s' will be ignored." % (f))
+                else:
+                    field_ids.append(field_id.value)
+        if key == 'update_freq':
+            update_freq = int(val)
+        if key == 'max_keep_age':
+            max_keep_age = int(max_keep_age)
+        if key == 'max_keep_samples':
+            max_keep_samples = int(max_keep_samples)
+        if key == 'gpu_indexes':
+            gpu_indexes = [int(x) for x in node.values]
+
+    if embedded:
+        rdc_ip_port = None
+    g_reader = CollectdReader(rdc_ip_port, field_ids, update_freq*1000000,
+            max_keep_age, max_keep_samples, gpu_indexes, unauth)
+
+def read_callback(data=None):
+    global g_reader
+    g_reader.process()
+
+collectd.register_config(config_func)
+collectd.register_read(read_callback)

projects/rdc/python_binding/rdc_grafana_dashboard_example.json

@@ -0,0 +1,991 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "description": "Dashboard to monitor AMD GPUs using RDC",
+  "editable": true,
+  "gnetId": 11756,
+  "graphTooltip": 0,
+  "id": 4,
+  "iteration": 1599146807681,
+  "links": [],
+  "panels": [
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "prometheus",
+      "decimals": 0,
+      "editable": true,
+      "error": false,
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fill": 0,
+      "fillGradient": 0,
+      "grid": {},
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 26,
+      "interval": "1s",
+      "legend": {
+        "avg": false,
+        "current": true,
+        "max": true,
+        "min": true,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "maxPerRow": 6,
+      "nullPointMode": "connected",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pluginVersion": "6.7.3",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "repeat": "node",
+      "repeatDirection": "h",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": true,
+      "targets": [
+        {
+          "expr": "power_usage{instance=~\"$node.*\",gpu_index=\"0\"}",
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "{{short_instance}}:gpu0",
+          "metric": "",
+          "refId": "A",
+          "step": 1200,
+          "target": ""
+        },
+        {
+          "expr": "power_usage{instance=~\"$node.*\",gpu_index=\"1\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu1",
+          "refId": "B"
+        },
+        {
+          "expr": "power_usage{instance=~\"$node.*\",gpu_index=\"2\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu2",
+          "refId": "C"
+        },
+        {
+          "expr": "power_usage{instance=~\"$node.*\",gpu_index=\"3\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu3",
+          "refId": "D"
+        },
+        {
+          "expr": "power_usage{instance=~\"$node.*\",gpu_index=\"4\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu4",
+          "refId": "E"
+        },
+        {
+          "expr": "power_usage{instance=~\"$node.*\",gpu_index=\"5\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu5",
+          "refId": "F"
+        },
+        {
+          "expr": "power_usage{instance=~\"$node.*\",gpu_index=\"6\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu6",
+          "refId": "G"
+        },
+        {
+          "expr": "power_usage{instance=~\"$node.*\",gpu_index=\"7\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu7",
+          "refId": "H"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Average GPU Package Power (Watt)",
+      "tooltip": {
+        "msResolution": false,
+        "shared": true,
+        "sort": 2,
+        "value_type": "cumulative"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "prometheus",
+      "decimals": 0,
+      "editable": true,
+      "error": false,
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fill": 0,
+      "fillGradient": 0,
+      "grid": {},
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 7
+      },
+      "hiddenSeries": false,
+      "id": 45,
+      "interval": "1s",
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": true,
+        "min": true,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "maxPerRow": 6,
+      "nullPointMode": "connected",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "repeat": "node",
+      "repeatDirection": "h",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "gpu_util{instance=~\"$node.*\",gpu_index=\"1\"}",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "{{short_instance}}:gpu1",
+          "metric": "",
+          "refId": "A",
+          "step": 1200,
+          "target": ""
+        },
+        {
+          "expr": "gpu_util{instance=~\"$node.*\",gpu_index=\"2\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu2",
+          "refId": "B"
+        },
+        {
+          "expr": "gpu_util{instance=~\"$node.*\",gpu_index=\"3\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu3",
+          "refId": "C"
+        },
+        {
+          "expr": "gpu_util{instance=~\"$node.*\",gpu_index=\"4\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu4",
+          "refId": "D"
+        },
+        {
+          "expr": "gpu_util{instance=~\"$node.*\",gpu_index=\"5\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu5",
+          "refId": "E"
+        },
+        {
+          "expr": "gpu_util{instance=~\"$node.*\",gpu_index=\"6\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu6",
+          "refId": "F"
+        },
+        {
+          "expr": "gpu_util{instance=~\"$node.*\",gpu_index=\"7\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu7",
+          "refId": "G"
+        },
+        {
+          "expr": "gpu_util{instance=~\"$node.*\",gpu_index=\"0\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu0",
+          "refId": "H"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "GPU Usage (%)",
+      "tooltip": {
+        "msResolution": false,
+        "shared": true,
+        "sort": 0,
+        "value_type": "cumulative"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "prometheus",
+      "decimals": 0,
+      "editable": true,
+      "error": false,
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fill": 0,
+      "fillGradient": 0,
+      "grid": {},
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 15
+      },
+      "hiddenSeries": false,
+      "id": 27,
+      "interval": "1s",
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": true,
+        "min": true,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "maxPerRow": 6,
+      "nullPointMode": "connected",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "repeat": "node",
+      "repeatDirection": "h",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "gpu_clock{instance=~\"$node.*\",gpu_index=\"1\"}",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "{{short_instance}}:gpu1",
+          "metric": "",
+          "refId": "A",
+          "step": 1200,
+          "target": ""
+        },
+        {
+          "expr": "gpu_clock{instance=~\"$node.*\",gpu_index=\"2\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu2",
+          "refId": "B"
+        },
+        {
+          "expr": "gpu_clock{instance=~\"$node.*\",gpu_index=\"3\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu3",
+          "refId": "C"
+        },
+        {
+          "expr": "gpu_clock{instance=~\"$node.*\",gpu_index=\"4\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu4",
+          "refId": "D"
+        },
+        {
+          "expr": "gpu_clock{instance=~\"$node.*\",gpu_index=\"5\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu5",
+          "refId": "E"
+        },
+        {
+          "expr": "gpu_clock{instance=~\"$node.*\",gpu_index=\"6\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu6",
+          "refId": "F"
+        },
+        {
+          "expr": "gpu_clock{instance=~\"$node.*\",gpu_index=\"7\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu7",
+          "refId": "G"
+        },
+        {
+          "expr": "gpu_clock{instance=~\"$node.*\",gpu_index=\"0\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu0",
+          "refId": "H"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "GPU Clock Speed (MHz)",
+      "tooltip": {
+        "msResolution": false,
+        "shared": true,
+        "sort": 0,
+        "value_type": "cumulative"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "prometheus",
+      "decimals": 0,
+      "description": "The GPU temperature in degree",
+      "editable": true,
+      "error": false,
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fill": 0,
+      "fillGradient": 0,
+      "grid": {},
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 23
+      },
+      "hiddenSeries": false,
+      "id": 86,
+      "interval": "1s",
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "maxPerRow": 6,
+      "nullPointMode": "connected",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "repeat": "node",
+      "repeatDirection": "h",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "gpu_temp{instance=~\"$node.*\",gpu_index=\"0\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu0  - Allocated",
+          "refId": "I"
+        },
+        {
+          "expr": "gpu_temp{instance=~\"$node.*\",gpu_index=\"7\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu7  - Allocated",
+          "refId": "J"
+        },
+        {
+          "expr": "gpu_temp{instance=~\"$node.*\",gpu_index=\"6\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu6  - Allocated",
+          "refId": "K"
+        },
+        {
+          "expr": "gpu_temp{instance=~\"$node.*\",gpu_index=\"5\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu5  - Allocated",
+          "refId": "L"
+        },
+        {
+          "expr": "gpu_temp{instance=~\"$node.*\",gpu_index=\"4\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu4  - Allocated",
+          "refId": "M"
+        },
+        {
+          "expr": "gpu_temp{instance=~\"$node.*\",gpu_index=\"3\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu3  - Allocated",
+          "refId": "N"
+        },
+        {
+          "expr": "gpu_temp{instance=~\"$node.*\",gpu_index=\"2\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu2  - Allocated",
+          "refId": "O"
+        },
+        {
+          "expr": "gpu_temp{instance=~\"$node.*\",gpu_index=\"1\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu1  - Allocated",
+          "refId": "P"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "GPU Temperature (Celsius)",
+      "tooltip": {
+        "msResolution": false,
+        "shared": true,
+        "sort": 0,
+        "value_type": "cumulative"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "prometheus",
+      "decimals": 0,
+      "description": "the amount of total available and allocated VRAM",
+      "editable": true,
+      "error": false,
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fill": 0,
+      "fillGradient": 0,
+      "grid": {},
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 23
+      },
+      "hiddenSeries": false,
+      "id": 65,
+      "interval": "1s",
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "maxPerRow": 6,
+      "nullPointMode": "connected",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "repeat": "node",
+      "repeatDirection": "h",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "gpu_memory_usage{instance=~\"$node.*\",gpu_index=\"0\"} / 1024",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu0  - Allocated",
+          "refId": "I"
+        },
+        {
+          "expr": "gpu_memory_usage{instance=~\"$node.*\",gpu_index=\"7\"} / 1024",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu7  - Allocated",
+          "refId": "J"
+        },
+        {
+          "expr": "gpu_memory_usage{instance=~\"$node.*\",gpu_index=\"6\"} / 1024",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu6  - Allocated",
+          "refId": "K"
+        },
+        {
+          "expr": "gpu_memory_usage{instance=~\"$node.*\",gpu_index=\"5\"} / 1024",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu5  - Allocated",
+          "refId": "L"
+        },
+        {
+          "expr": "gpu_memory_usage{instance=~\"$node.*\",gpu_index=\"4\"} / 1024",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu4  - Allocated",
+          "refId": "M"
+        },
+        {
+          "expr": "gpu_memory_usage{instance=~\"$node.*\",gpu_index=\"3\"} / 1024",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu3  - Allocated",
+          "refId": "N"
+        },
+        {
+          "expr": "gpu_memory_usage{instance=~\"$node.*\",gpu_index=\"2\"} / 1024",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu2  - Allocated",
+          "refId": "O"
+        },
+        {
+          "expr": "gpu_memory_usage{instance=~\"$node.*\",gpu_index=\"1\"} / 1024",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu1  - Allocated",
+          "refId": "P"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "GPU Memory Allocation (GB)",
+      "tooltip": {
+        "msResolution": false,
+        "shared": true,
+        "sort": 0,
+        "value_type": "cumulative"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "prometheus",
+      "decimals": 0,
+      "description": "indicate how busy the respective mem blocks are",
+      "editable": true,
+      "error": false,
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fill": 0,
+      "fillGradient": 0,
+      "grid": {},
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 31
+      },
+      "hiddenSeries": false,
+      "id": 64,
+      "interval": "1s",
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": true,
+        "min": true,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "maxPerRow": 6,
+      "nullPointMode": "connected",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "repeat": "node",
+      "repeatDirection": "h",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "gpu_memory_usage{instance=~\"$node.*\",gpu_index=\"1\"}*100/gpu_memory_total{instance=~\"$node.*\",gpu_index=\"1\"}",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "{{short_instance}}:gpu1",
+          "metric": "",
+          "refId": "A",
+          "step": 1200,
+          "target": ""
+        },
+        {
+          "expr": "gpu_memory_usage{instance=~\"$node.*\",gpu_index=\"2\"}*100/gpu_memory_total{instance=~\"$node.*\",gpu_index=\"2\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu2",
+          "refId": "B"
+        },
+        {
+          "expr": "gpu_memory_usage{instance=~\"$node.*\",gpu_index=\"3\"}*100/gpu_memory_total{instance=~\"$node.*\",gpu_index=\"3\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu3",
+          "refId": "C"
+        },
+        {
+          "expr": "gpu_memory_usage{instance=~\"$node.*\",gpu_index=\"4\"}*100/gpu_memory_total{instance=~\"$node.*\",gpu_index=\"4\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu4",
+          "refId": "D"
+        },
+        {
+          "expr": "gpu_memory_usage{instance=~\"$node.*\",gpu_index=\"5\"}*100/gpu_memory_total{instance=~\"$node.*\",gpu_index=\"5\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu5",
+          "refId": "E"
+        },
+        {
+          "expr": "gpu_memory_usage{instance=~\"$node.*\",gpu_index=\"6\"}*100/gpu_memory_total{instance=~\"$node.*\",gpu_index=\"6\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu6",
+          "refId": "F"
+        },
+        {
+          "expr": "gpu_memory_usage{instance=~\"$node.*\",gpu_index=\"7\"}*100/gpu_memory_total{instance=~\"$node.*\",gpu_index=\"7\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu7",
+          "refId": "G"
+        },
+        {
+          "expr": "gpu_memory_usage{instance=~\"$node.*\",gpu_index=\"0\"}*100/gpu_memory_total{instance=~\"$node.*\",gpu_index=\"0\"}",
+          "interval": "",
+          "legendFormat": "{{short_instance}}:gpu0",
+          "refId": "H"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "GPU Memory Activity Level (%)",
+      "tooltip": {
+        "msResolution": false,
+        "shared": true,
+        "sort": 0,
+        "value_type": "cumulative"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    }
+  ],
+  "refresh": false,
+  "schemaVersion": 25,
+  "style": "dark",
+  "tags": [
+    "IB"
+  ],
+  "templating": {
+    "list": [
+      {
+        "allFormat": "glob",
+        "allValue": null,
+        "current": {
+          "selected": true,
+          "tags": [],
+          "text": "",
+          "value": []
+        },
+        "datasource": "prometheus",
+        "definition": "label_values(instance)",
+        "hide": 0,
+        "includeAll": false,
+        "label": "Host",
+        "multi": true,
+        "multiFormat": "regex values",
+        "name": "node",
+        "options": [],
+        "query": "label_values(instance)",
+        "refresh": 1,
+        "regex": "/(.*):.*/",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {
+    "now": true,
+    "refresh_intervals": [
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "1m",
+      "2m",
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "browser",
+  "title": "ROCm Data Center tool V1.0",
+  "uid": "thisIsAuniqueID",
+  "version": 21
+}

projects/rdc/python_binding/rdc_prometheus.py

@@ -0,0 +1,142 @@
+import argparse
+import os
+from RdcReader import RdcReader
+from RdcUtil import RdcUtil
+from rdc_bootstrap import *
+from prometheus_client import start_http_server, Gauge, Counter, Info, REGISTRY, PROCESS_COLLECTOR, PLATFORM_COLLECTOR
+
+os.environ['PROMETHEUS_DISABLE_CREATED_SERIES'] = "True"
+
+default_field_ids = [
+        rdc_field_t.RDC_FI_GPU_MEMORY_USAGE,
+        rdc_field_t.RDC_FI_GPU_MEMORY_TOTAL,
+        rdc_field_t.RDC_FI_POWER_USAGE,
+        rdc_field_t.RDC_FI_GPU_CLOCK,
+        rdc_field_t.RDC_FI_GPU_UTIL,
+        rdc_field_t.RDC_FI_GPU_TEMP,
+        rdc_field_t.RDC_FI_PROF_ACTIVE_CYCLES,
+        rdc_field_t.RDC_FI_PROF_ACTIVE_WAVES,
+        rdc_field_t.RDC_FI_PROF_OCCUPANCY_PERCENT,
+]
+
+class PrometheusReader(RdcReader):
+    def __init__(self, rdc_ip_port, field_ids, update_freq, max_keep_age, max_keep_samples,
+            gpu_indexes, rdc_unauth, enable_plugin_monitoring):
+        group_name = "rdc_prometheus_plugin_group"
+        field_group_name = "rdc_prometheus_plugin_fieldgroup"
+        if rdc_unauth:
+            RdcReader.__init__(self, ip_port = rdc_ip_port, field_ids = field_ids, update_freq=update_freq,
+                    max_keep_age = max_keep_age, max_keep_samples = max_keep_samples,
+                    gpu_indexes = gpu_indexes, field_group_name = field_group_name, gpu_group_name = group_name, root_ca = None)
+        else:
+             RdcReader.__init__(self, ip_port = rdc_ip_port, field_ids = field_ids, update_freq=update_freq,
+                    max_keep_age = max_keep_age, max_keep_samples = max_keep_samples,
+                    gpu_indexes = gpu_indexes, field_group_name = field_group_name, gpu_group_name = group_name)
+
+        # Supress internal metrics from prometheus_client
+        if enable_plugin_monitoring == False:
+            REGISTRY.unregister(PROCESS_COLLECTOR)
+            REGISTRY.unregister(PLATFORM_COLLECTOR)
+
+        # Create the metrics
+        self.gauges = {}
+        self.counters = {}
+        self.infos = {}
+
+        for fid in self.field_ids:
+            field_name = self.rdc_util.field_id_string(fid)
+
+            rdc_metric_type = rdc_field_t.get_rdc_metric_type(rdc_field_t.get_field_name(fid))
+
+            field_name = field_name.lower()
+
+
+            if rdc_metric_type == 1:
+                self.gauges[fid] = Gauge(field_name, field_name, labelnames=['gpu_index'])
+            elif rdc_metric_type == 2:
+                self.counters[fid] = Counter(field_name, field_name, labelnames=['gpu_index'])
+            else:
+                self.infos[fid] = Info(field_name, field_name, labelnames=['gpu_index'])
+            
+
+
+    def handle_field(self, gpu_index, value):
+        gpu_label = gpu_index
+        if value.field_id.value in self.gauges:
+            self.gauges[value.field_id.value].labels(gpu_label).set(value.value.l_int)
+        elif value.field_id.value in self.counters:
+            self.counters[value.field_id.value].labels(gpu_label).inc(value.value.l_int)
+        else:
+            self.infos[value.field_id.value].labels(gpu_label).info({'gpu_label': self.process_value(value)})
+
+    def process_value(self, value):
+        if value.type.value == rdc_field_type_t.INTEGER:
+            return str(value.value.l_int)
+        elif value.type.value == rdc_field_type_t.DOUBLE:
+            return str(value.value.d_float)
+        elif value.type.value == rdc_field_type_t.STRING:
+            return value.value.str.decode('utf-8', 'ignore')
+        elif value.type.value == rdc_field_type_t.BLOB:
+            return value.value.str.hex()
+        else:
+            return "unknown"
+
+def get_field_ids(args):
+    field_ids = []
+
+    field_id_str=[]
+    if args.rdc_fields:
+        field_id_str=args.rdc_fields
+    elif args.rdc_fields_file:
+        try:
+            with open(args.rdc_fields_file) as fi:
+                content = fi.readlines()
+            field_id_str = [x.strip() for x in content]
+        except Exception as e:
+             print("Fail to read " + args.rdc_fields_file + ":" + str(e))
+
+    if len(field_id_str)> 0 :
+        for f in field_id_str:
+            field_id = rdc.get_field_id_from_name(str.encode(f))
+            if field_id.value == rdc_field_t.RDC_FI_INVALID:
+                print("Invalid field '%s' will be ignored." % (f))
+            else:
+                field_ids.append(field_id.value)
+        return field_ids
+
+    return default_field_ids
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='RDC Prometheus plugin.')
+    parser.add_argument('--listen_port', default=5000, type=int, help='The listen port of the plugin (default: 5000)')
+    parser.add_argument('--rdc_embedded', default=False, action='store_true', help='Run RDC in embedded mode (default: standalone mode)')
+    parser.add_argument('--rdc_ip_port' , default='localhost:50051', help='The rdcd IP and port in standalone mode (default: localhost:50051)')
+    parser.add_argument('--rdc_unauth', default=False, action='store_true', help='Set this option if the rdcd is running with unauth in standalone mode (default: false)')
+    parser.add_argument('--rdc_update_freq', default=10, help='The fields update frequency in seconds (default: 10)')
+    parser.add_argument('--rdc_max_keep_age', default=3600, help='The max keep age of the fields in seconds (default: 3600)')
+    parser.add_argument('--rdc_max_keep_samples', default=1000, help='The max samples to keep for each field in the cache (default: 1000)')
+    parser.add_argument('--rdc_fields', default=None, nargs='+', help='The list of fields name needs to be watched, for example, " --rdc_fields RDC_FI_GPU_TEMP RDC_FI_POWER_USAGE " (default: predefined fields in the plugin)')
+    parser.add_argument('--rdc_fields_file', default=None, help='The list of fields name can also be read from a file with each field name in a separated line (default: None)')
+    parser.add_argument('--rdc_gpu_indexes',  default=None, nargs='+', help='The list of GPUs to be watched (default: All GPUs)')
+    parser.add_argument('--enable_plugin_monitoring', default=False, action='store_true', help = 'Set this option to collect process metrics of the plugin itself (default: false)')
+
+    args = parser.parse_args()
+
+    field_ids = get_field_ids(args)
+    rdc_ip_port = args.rdc_ip_port
+    if args.rdc_embedded:
+        rdc_ip_port = None
+    if args.rdc_gpu_indexes != None:
+        for i in range(0, len(args.rdc_gpu_indexes)):
+            args.rdc_gpu_indexes[i] = int(args.rdc_gpu_indexes[i])
+
+    reader = PrometheusReader(rdc_ip_port, field_ids, args.rdc_update_freq*1000000,
+            args.rdc_max_keep_age, args.rdc_max_keep_samples,
+            args.rdc_gpu_indexes, args.rdc_unauth, args.enable_plugin_monitoring)
+    start_http_server(args.listen_port)
+    print("The RDC Prometheus plugin listen at port %d" % (args.listen_port))
+    time.sleep(3)
+    while True:
+        reader.process()
+        time.sleep(1)

projects/rdc/python_binding/rdc_prometheus_example.yml

@@ -0,0 +1,23 @@
+# global config
+global:
+  scrape_interval:     10s # Set the scrape interval to every 10 seconds. Default is every 1 minute.
+  evaluation_interval: 10s # Evaluate rules every 10 seconds. The default is every 1 minute.
+  # scrape_timeout is set to the global default (10s).
+
+# A scrape configuration where the endpoints to scrape will be defined at prometheus_targets.json:
+scrape_configs:
+  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+  - job_name: 'rdc'
+
+    # metrics_path defaults to '/metrics'
+    # scheme defaults to 'http'.
+
+    # Remove the port for display
+    relabel_configs:
+    - source_labels: [__address__]
+      regex: '([^:]+):\d+'
+      target_label: short_instance
+
+    file_sd_configs:
+    - files:
+        - 'prometheus_targets.json'

projects/rdc/python_binding/rdc_rest_api.py

@@ -0,0 +1,152 @@
+#
+# Copyright (C) Advanced Micro Devices. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+from flask import Flask, request, jsonify
+from RdcReader import RdcReader
+from RdcUtil import RdcUtil
+from rdc_bootstrap import *
+
+# Initialize Flask app
+app = Flask(__name__)
+
+# Initialize RDC Reader and Utilities for handling GPU queries
+rdc_reader = RdcReader(ip_port=None)
+rdc_util = RdcUtil()
+
+# Dictionary to store query criteria with query_id
+gpu_queries = {}
+
+# Endpoint to discover available GPUs
+@app.route('/rdc/discovery', methods=['GET'])
+def discover_gpus():
+    """Retrieve a list of available GPUs and their names."""
+    try:
+        gpu_indexes = rdc_util.get_all_gpu_indexes(rdc_reader.rdc_handle)
+        gpus = {}
+        for gpu in gpu_indexes:
+            device_attr = rdc_device_attributes_t()
+            rdc.rdc_device_get_attributes(rdc_reader.rdc_handle, gpu, device_attr)
+            gpus[gpu] = device_attr.device_name.decode('utf-8')  # Decode GPU name from bytes
+        return jsonify(gpus)
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
+# Endpoint to create a new query criteria
+@app.route('/rdc/query_criteria', methods=['POST'])
+def create_query_criteria():
+    """Define a new query criteria specifying GPU indices and metrics to monitor."""
+    try:
+        data = request.json
+        if not data or "metrics" not in data:
+            return jsonify({"error": "Invalid request payload"}), 400
+        
+        gpu_indexes = data.get("gpu_index", rdc_util.get_all_gpu_indexes(rdc_reader.rdc_handle))
+        metrics = data.get("metrics", [])
+        
+        # Create rdc group and fieldgroup
+        gpu_group_id, _ = rdc_util.create_gpu_group(rdc_reader.rdc_handle, b"query_gpu_group", gpu_indexes)
+        field_group_id, _ = rdc_util.create_field_group(rdc_reader.rdc_handle, b"query_field_group", [rdc.get_field_id_from_name(m.encode('utf-8')).value for m in metrics])
+
+        # Call rdc_field_watch to start fetching metrics into cache
+        result = rdc.rdc_field_watch(rdc_reader.rdc_handle, gpu_group_id, field_group_id, 1000000, 3600.0, 1000)
+        if rdc_status_t(result) != rdc_status_t.RDC_ST_OK:
+            return jsonify({"error": "Failed to watch fields"}), 500
+        
+        query_id = f"G-{gpu_group_id.value}-F-{field_group_id.value}"
+        gpu_queries[query_id] = {"gpu_index": gpu_indexes, "metrics": metrics, "query_id": query_id}
+        return jsonify({"query_id": query_id})
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
+# Endpoint to get all query criteria
+@app.route('/rdc/query_criteria', methods=['GET'])
+def get_all_query_criteria():
+    """Retrieve all stored query criteria for all GPUs."""
+    try:
+        query_id = request.args.get("query_id")
+        if query_id:
+            return jsonify(gpu_queries.get(query_id, {}))
+        return jsonify(list(gpu_queries.values()))
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
+# Endpoint to retrieve a specific query criteria
+@app.route('/rdc/query_criteria/<query_id>', methods=['GET'])
+def get_query_criteria(query_id):
+    """Retrieve query criteria based on a given query ID."""
+    try:
+        if query_id in gpu_queries:
+            return jsonify(gpu_queries[query_id])
+        return jsonify({"error": "Query ID not found"}), 404
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
+# Endpoint to delete a specific query criteria
+@app.route('/rdc/query_criteria/<query_id>', methods=['DELETE'])
+def delete_query_criteria(query_id):
+    """Delete a query criteria using its query ID."""
+    try:
+        if query_id in gpu_queries:
+            gpu_group_id = rdc_reader.field_group_id
+            field_group_id = rdc_reader.field_group_id
+            
+            # Call rdc_field_unwatch to stop fetching metrics
+            result = rdc.rdc_field_unwatch(rdc_reader.rdc_handle, gpu_group_id, field_group_id)
+            if rdc_status_t(result) != rdc_status_t.RDC_ST_OK:
+                return jsonify({"error": "Failed to unwatch fields"}), 500
+            
+            # Delete GPU and field groups
+            rdc.rdc_group_gpu_destroy(rdc_reader.rdc_handle, gpu_group_id)
+            rdc.rdc_group_field_destroy(rdc_reader.rdc_handle, field_group_id)
+            
+            # Remove the query from storage
+            del gpu_queries[query_id]
+            return jsonify({"message": "Deleted successfully"})
+        return jsonify({"error": "Query ID not found"}), 404
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
+# Endpoint to fetch GPU metrics for a specific query ID
+@app.route('/rdc/gpu_metrics/<query_id>', methods=['GET'])
+def get_gpu_metrics(query_id):
+    """Retrieve GPU metrics based on the query ID."""
+    try:
+        if query_id not in gpu_queries:
+            return jsonify({"error": "Query ID not found"}), 404
+        
+        query = gpu_queries[query_id]
+        gpu_metrics = []  # List to store GPU metric results
+        for gpu in query["gpu_index"]:
+            gpu_data = {"gpu_index": gpu}  # Store GPU index in the response
+            for metric in query["metrics"]:
+                field_id = rdc.get_field_id_from_name(metric.encode('utf-8')).value
+                value = rdc_field_value()
+                result = rdc.rdc_field_get_latest_value(rdc_reader.rdc_handle, gpu, field_id, value)
+                if rdc_status_t(result) == rdc_status_t.RDC_ST_OK:
+                    gpu_data[metric] = value.value.l_int  # Store metric value
+            gpu_metrics.append(gpu_data)  # Append GPU data to results
+        return jsonify(gpu_metrics)
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
+# Main entry point to start the Flask server
+if __name__ == '__main__':
+    # Runs the API server, making it accessible on all network interfaces
+    app.run(host='0.0.0.0', port=50052)