From 459929b5c0f71b87dc3f7c432699aaf1fed564ab Mon Sep 17 00:00:00 2001 From: Peter Park Date: Thu, 26 Sep 2024 10:44:01 -0400 Subject: [PATCH 1/2] Add LD_LIBRARY_PATH note to rocm.docs pages https://github.com/ROCm/rocm_smi_lib/pull/197 https://advanced-micro-devices-demo--197.com.readthedocs.build/projects/rocm_smi_lib/en/197/ Change-Id: I64386a4f364e40ce61ad9963376d2686db2aa36d [ROCm/rocm_smi_lib commit: b7221c64b02782d1def4423142324020e1854260] --- projects/rocm-smi-lib/docs/how-to/use-cpp.rst | 28 +- .../rocm-smi-lib/docs/how-to/use-python.md | 454 --------------- .../rocm-smi-lib/docs/how-to/use-python.rst | 519 ++++++++++++++++++ projects/rocm-smi-lib/docs/index.rst | 14 +- projects/rocm-smi-lib/docs/sphinx/_toc.yml.in | 2 - 5 files changed, 546 insertions(+), 471 deletions(-) delete mode 100644 projects/rocm-smi-lib/docs/how-to/use-python.md create mode 100644 projects/rocm-smi-lib/docs/how-to/use-python.rst diff --git a/projects/rocm-smi-lib/docs/how-to/use-cpp.rst b/projects/rocm-smi-lib/docs/how-to/use-cpp.rst index f16bb1c35d..1ebfe7b5cd 100644 --- a/projects/rocm-smi-lib/docs/how-to/use-cpp.rst +++ b/projects/rocm-smi-lib/docs/how-to/use-cpp.rst @@ -1,23 +1,33 @@ - .. meta:: - :description: Using ROCm SMI - :keywords: install, SMI, library, api, AMD, ROCm + :description: Learn about using the ROCm SMI library with C++. + :keywords: install, SMI, library, api, cpp, system management interface +*********************** +Using ROCm SMI with C++ +*********************** -Using C++ in ROCm SMI -********************* +Installation +============ -Device indices ---------------- +Follow the installation procedure for rocm_smi_lib. Refer to the :doc:`installation section <../install/install>`. + +.. note:: + + ``hipcc`` and other compilers will not automatically link in the ``librocm_smi64`` dynamic library. To ensure the + ``librocm_smi64.so`` can be located, you must either set the ``LD_LIBRARY_PATH`` environment variable to the + directory containing ``librocm_smi64.so`` (usually ``/opt/rocm/lib``) or pass the ``-lrocm_smi64`` flag to the compiler. + +Device indices +============== Many of the functions in the library take a "device index". The device index is a number greater than or equal to 0, and less than the number of devices detected, as determined by `rsmi_num_monitor_devices()`. The index is used to distinguish the detected devices from one another. It is important to note that a device may end up with a different index after a reboot, so an index should not be relied upon to be constant over reboots. Hello ROCm SMI ================ -The only required ROCm-SMI call for any program that wants to use ROCm-SMI is the `rsmi_init()` call. This call initializes some internal data structures that will be used by subsequent ROCm-SMI calls. +The only required ROCm-SMI call for any program that wants to use ROCm-SMI is the ``rsmi_init()`` call. This call initializes some internal data structures that will be used by subsequent ROCm-SMI calls. -When ROCm-SMI is no longer being used, `rsmi_shut_down()` should be called. This provides a way to do any releasing of resources that ROCm-SMI may have held. In many cases, this may have no effect, but may be necessary in future versions of the library. +When ROCm-SMI is no longer being used, ``rsmi_shut_down()`` should be called. This provides a way to do any releasing of resources that ROCm-SMI may have held. In many cases, this may have no effect, but may be necessary in future versions of the library. A simple "Hello World" type program that displays the device ID of detected devices would look like this: diff --git a/projects/rocm-smi-lib/docs/how-to/use-python.md b/projects/rocm-smi-lib/docs/how-to/use-python.md deleted file mode 100644 index 085ef4c0e6..0000000000 --- a/projects/rocm-smi-lib/docs/how-to/use-python.md +++ /dev/null @@ -1,454 +0,0 @@ - -# Using Python in ROCm SMI - -This tool acts as a command line interface for manipulating and monitoring the amdgpu kernel, and is intended to replace and deprecate the existing rocm_smi.py CLI tool. -It uses Ctypes to call the rocm_smi_lib API. Recommended: At least one AMD GPU with ROCm driver installed - -Required: ROCm SMI library installed (librocm_smi64) - -## Installation - -Follow the installation procedure for rocm_smi_lib. Refer to the [installation](../install/install.rst) section. - -LD_LIBRARY_PATH must be set to the folder containing librocm_smi64. - -## Version - -The SMI will report two "versions", ROCM-SMI version and other is ROCM-SMI-LIB version. - -- ROCM-SMI version is the CLI/tool version number with commit ID appended after + sign. -- ROCM-SMI-LIB version is the library package version number. - -``` -ROCM-SMI version: 2.0.0+8e78352 -ROCM-SMI-LIB version: 6.1.0 -``` - -## Usage - -For detailed and up to date usage information, we recommend consulting the help: - - /opt/rocm/bin/rocm-smi -h - -For convenience purposes, following is the output from the -h flag: - -``` -/opt/rocm/bin/rocm-smi -h -usage: rocm-smi [-h] [-V] [-d DEVICE [DEVICE ...]] [--alldevices] [--showhw] [-a] [-i] [-v] [-e [EVENT [EVENT ...]]] - [--showdriverversion] [--showtempgraph] [--showfwinfo [BLOCK [BLOCK ...]]] [--showmclkrange] - [--showmemvendor] [--showsclkrange] [--showproductname] [--showserial] [--showuniqueid] - [--showvoltagerange] [--showbus] [--showpagesinfo] [--showpendingpages] [--showretiredpages] - [--showunreservablepages] [-f] [-P] [-t] [-u] [--showmemuse] [--showvoltage] [-b] [-c] [-g] [-l] [-M] - [-m] [-o] [-p] [-S] [-s] [--showmeminfo TYPE [TYPE ...]] [--showpids [VERBOSE]] - [--showpidgpus [SHOWPIDGPUS [SHOWPIDGPUS ...]]] [--showreplaycount] - [--showrasinfo [SHOWRASINFO [SHOWRASINFO ...]]] [--showvc] [--showxgmierr] [--showtopo] - [--showtopoaccess] [--showtopoweight] [--showtopohops] [--showtopotype] [--showtoponuma] - [--showenergycounter] [--shownodesbw] [--showcomputepartition] [--showmemorypartition] [-r] - [--resetfans] [--resetprofile] [--resetpoweroverdrive] [--resetxgmierr] [--resetperfdeterminism] - [--resetcomputepartition] [--resetmemorypartition] [--setclock TYPE LEVEL] [--setsclk LEVEL [LEVEL ...]] - [--setmclk LEVEL [LEVEL ...]] [--setpcie LEVEL [LEVEL ...]] [--setslevel SCLKLEVEL SCLK SVOLT] - [--setmlevel MCLKLEVEL MCLK MVOLT] [--setvc POINT SCLK SVOLT] [--setsrange SCLKMIN SCLKMAX] - [--setextremum min|max sclk|mclk CLK] [--setmrange MCLKMIN MCLKMAX] [--setfan LEVEL] - [--setperflevel LEVEL] [--setoverdrive %] [--setmemoverdrive %] [--setpoweroverdrive WATTS] - [--setprofile SETPROFILE] [--setperfdeterminism SCLK] - [--setcomputepartition {CPX,SPX,DPX,TPX,QPX,cpx,spx,dpx,tpx,qpx}] - [--setmemorypartition {NPS1,NPS2,NPS4,NPS8,nps1,nps2,nps4,nps8}] [--rasenable BLOCK ERRTYPE] - [--rasdisable BLOCK ERRTYPE] [--rasinject BLOCK] [--gpureset] [--load FILE | --save FILE] - [--autorespond RESPONSE] [--loglevel LEVEL] [--json] [--csv] - -AMD ROCm System Management Interface | ROCM-SMI version: 2.0.0+8e78352 - -optional arguments: - -h, --help show this help message and exit - --gpureset Reset specified GPU (One GPU must be specified) - --load FILE Load Clock, Fan, Performance and Profile settings - from FILE - --save FILE Save Clock, Fan, Performance and Profile settings to - FILE - - -V, --version Show version information - - -d DEVICE [DEVICE ...], --device DEVICE [DEVICE ...] Execute command on specified device - -Display Options: - --alldevices - --showhw Show Hardware details - -a, --showallinfo Show Temperature, Fan and Clock values - -Topology: - -i, --showid Show DEVICE ID - -v, --showvbios Show VBIOS version - -e [EVENT [EVENT ...]], --showevents [EVENT [EVENT ...]] Show event list - --showdriverversion Show kernel driver version - --showtempgraph Show Temperature Graph - --showfwinfo [BLOCK [BLOCK ...]] Show FW information - --showmclkrange Show mclk range - --showmemvendor Show GPU memory vendor - --showsclkrange Show sclk range - --showproductname Show SKU/Vendor name - --showserial Show GPU's Serial Number - --showuniqueid Show GPU's Unique ID - --showvoltagerange Show voltage range - --showbus Show PCI bus number - -Pages information: - --showpagesinfo Show retired, pending and unreservable pages - --showpendingpages Show pending retired pages - --showretiredpages Show retired pages - --showunreservablepages Show unreservable pages - -Hardware-related information: - -f, --showfan Show current fan speed - -P, --showpower Show current Average or Socket Graphics Package Power - Consumption - -t, --showtemp Show current temperature - -u, --showuse Show current GPU use - --showmemuse Show current GPU memory used - --showvoltage Show current GPU voltage - -Software-related/controlled information: - -b, --showbw Show estimated PCIe use - -c, --showclocks Show current clock frequencies - -g, --showgpuclocks Show current GPU clock frequencies - -l, --showprofile Show Compute Profile attributes - -M, --showmaxpower Show maximum graphics package power this GPU will - consume - -m, --showmemoverdrive Show current GPU Memory Clock OverDrive level - -o, --showoverdrive Show current GPU Clock OverDrive level - -p, --showperflevel Show current DPM Performance Level - -S, --showclkvolt Show supported GPU and Memory Clocks and Voltages - -s, --showclkfrq Show supported GPU and Memory Clock - --showmeminfo TYPE [TYPE ...] Show Memory usage information for given block(s) TYPE - --showpids [VERBOSE] Show current running KFD PIDs (pass details to - VERBOSE for detailed information) - --showpidgpus [SHOWPIDGPUS [SHOWPIDGPUS ...]] Show GPUs used by specified KFD PIDs (all if no arg - given) - --showreplaycount Show PCIe Replay Count - --showrasinfo [SHOWRASINFO [SHOWRASINFO ...]] Show RAS enablement information and error counts for - the specified block(s) (all if no arg given) - --showvc Show voltage curve - --showxgmierr Show XGMI error information since last read - --showtopo Show hardware topology information - --showtopoaccess Shows the link accessibility between GPUs - --showtopoweight Shows the relative weight between GPUs - --showtopohops Shows the number of hops between GPUs - --showtopotype Shows the link type between GPUs - --showtoponuma Shows the numa nodes - --showenergycounter Energy accumulator that stores amount of energy - consumed - --shownodesbw Shows the numa nodes - --showcomputepartition Shows current compute partitioning - --showmemorypartition Shows current memory partition - -Set options: - --setclock TYPE LEVEL Set Clock Frequency Level(s) for specified clock - (requires manual Perf level) - --setsclk LEVEL [LEVEL ...] Set GPU Clock Frequency Level(s) (requires manual - Perf level) - --setmclk LEVEL [LEVEL ...] Set GPU Memory Clock Frequency Level(s) (requires - manual Perf level) - --setpcie LEVEL [LEVEL ...] Set PCIE Clock Frequency Level(s) (requires manual - Perf level) - --setslevel SCLKLEVEL SCLK SVOLT Change GPU Clock frequency (MHz) and Voltage (mV) for - a specific Level - --setmlevel MCLKLEVEL MCLK MVOLT Change GPU Memory clock frequency (MHz) and Voltage - for (mV) a specific Level - --setvc POINT SCLK SVOLT Change SCLK Voltage Curve (MHz mV) for a specific - point - --setsrange SCLKMIN SCLKMAX Set min and max SCLK speed - --setextremum min|max sclk|mclk CLK Set min/max of SCLK/MCLK speed - --setmrange MCLKMIN MCLKMAX Set min and max MCLK speed - --setfan LEVEL Set GPU Fan Speed (Level or %) - --setperflevel LEVEL Set Performance Level - --setoverdrive % Set GPU OverDrive level (requires manual|high Perf - level) - --setmemoverdrive % Set GPU Memory Overclock OverDrive level (requires - manual|high Perf level) - --setpoweroverdrive WATTS Set the maximum GPU power using Power OverDrive in - Watts - --setprofile SETPROFILE Specify Power Profile level (#) or a quoted string of - CUSTOM Profile attributes "# # # #..." (requires - manual Perf level) - --setperfdeterminism SCLK Set clock frequency limit to get minimal performance - variation - --setcomputepartition {CPX,SPX,DPX,TPX,QPX,cpx,spx,dpx,tpx,qpx} Set compute partition - --setmemorypartition {NPS1,NPS2,NPS4,NPS8,nps1,nps2,nps4,nps8} Set memory partition - --rasenable BLOCK ERRTYPE Enable RAS for specified block and error type - --rasdisable BLOCK ERRTYPE Disable RAS for specified block and error type - --rasinject BLOCK Inject RAS poison for specified block (ONLY WORKS ON - UNSECURE BOARDS) - -Reset options: - -r, --resetclocks Reset clocks and OverDrive to default - --resetfans Reset fans to automatic (driver) control - --resetprofile Reset Power Profile back to default - --resetpoweroverdrive Set the maximum GPU power back to the device deafult - state - --resetxgmierr Reset XGMI error count - --resetperfdeterminism Disable performance determinism - --resetcomputepartition Resets to boot compute partition state - --resetmemorypartition Resets to boot memory partition state - -Auto-response options: - --autorespond RESPONSE Response to automatically provide for all prompts - (NOT RECOMMENDED) - -Output options: - --loglevel LEVEL How much output will be printed for what program is - doing, one of debug/info/warning/error/critical - --json Print output in JSON format - --csv Print output in CSV format -``` - -## Detailed Option Descriptions -`--setextremum ` -Provided ASIC support, users can now set a maximum or minimum sclk or mclk value through our Python CLI tool (`rocm-smi --setextremum max sclk 1500`). See example below. - -```shell -$ sudo /opt/rocm/bin/rocm-smi --setextremum max sclk 2100 - -============================ ROCm System Management Interface ============================ - - ******WARNING****** - - Operating your AMD GPU outside of official AMD specifications or outside of - factory settings, including but not limited to the conducting of overclocking, - over-volting or under-volting (including use of this interface software, - even if such software has been directly or indirectly provided by AMD or otherwise - affiliated in any way with AMD), may cause damage to your AMD GPU, system components - and/or result in system failure, as well as cause other problems. - DAMAGES CAUSED BY USE OF YOUR AMD GPU OUTSIDE OF OFFICIAL AMD SPECIFICATIONS OR - OUTSIDE OF FACTORY SETTINGS ARE NOT COVERED UNDER ANY AMD PRODUCT WARRANTY AND - MAY NOT BE COVERED BY YOUR BOARD OR SYSTEM MANUFACTURER'S WARRANTY. - Use this utility with caution. - -Do you accept these terms? [y/N] y -================================ Set Valid sclk Extremum ================================= -GPU[0] : Successfully set max sclk to 2100(MHz) -GPU[1] : Successfully set max sclk to 2100(MHz) -GPU[2] : Successfully set max sclk to 2100(MHz) -GPU[3] : Successfully set max sclk to 2100(MHz) -================================== End of ROCm SMI Log =================================== -``` - ---setsclk/--setmclk # [# # ...]: - This allows you to set a mask for the levels. For example, if a GPU has 8 clock levels, - you can set a mask to use levels 0, 5, 6 and 7 with --setsclk 0 5 6 7 . This will only - use the base level, and the top 3 clock levels. This will allow you to keep the GPU at - base level when there is no GPU load, and the top 3 levels when the GPU load increases. - - NOTES: - The clock levels will change dynamically based on GPU load based on the default - Compute and Graphics profiles. The thresholds and delays for a custom mask cannot - be controlled through the SMI tool - - This flag automatically sets the Performance Level to "manual" as the mask is not - applied when the Performance level is set to auto - ---setfan LEVEL: - This sets the fan speed to a value ranging from 0 to maxlevel, or from 0%-100% - - If the level ends with a %, the fan speed is calculated as pct*maxlevel/100 - (maxlevel is usually 255, but is determined by the ASIC) - - NOTE: While the hardware is usually capable of overriding this value when required, it is - recommended to not set the fan level lower than the default value for extended periods - of time - ---setperflevel LEVEL: - This lets you use the pre-defined Performance Level values for clocks and power profile, which can include: - auto (Automatically change values based on GPU workload) - low (Keep values low, regardless of workload) - high (Keep values high, regardless of workload) - manual (Only use values defined by --setsclk and --setmclk) - ---setoverdrive/--setmemoverdrive #: - ***DEPRECATED IN NEWER KERNEL VERSIONS (use --setslevel/--setmlevel instead)*** - This sets the percentage above maximum for the max Performance Level. - For example, --setoverdrive 20 will increase the top sclk level by 20%, similarly - --setmemoverdrive 20 will increase the top mclk level by 20%. Thus if the maximum - clock level is 1000MHz, then --setoverdrive 20 will increase the maximum clock to 1200MHz - - NOTES: - This option can be used in conjunction with the --setsclk/--setmclk mask - - Operating the GPU outside of specifications can cause irreparable damage to your hardware - Observe the warning displayed when using this option - - This flag automatically sets the clock to the highest level, as only the highest level is - increased by the OverDrive value - ---setpoweroverdrive/--resetpoweroverdrive #: - This allows users to change the maximum power available to a GPU package. - The input value is in Watts. This limit is enforced by the hardware, and - some cards allow users to set it to a higher value than the default that - ships with the GPU. This Power OverDrive mode allows the GPU to run at - higher frequencies for longer periods of time, though this may mean the - GPU uses more power than it is allowed to use per power supply - specifications. Each GPU has a model-specific maximum Power OverDrive that - is will take; attempting to set a higher limit than that will cause this - command to fail. - - NOTES: - Operating the GPU outside of specifications can cause irreparable damage to your hardware - Observe the warning displayed when using this option - ---setprofile SETPROFILE: - The Compute Profile accepts 1 or n parameters, either the Profile to select (see --showprofile for a list - of preset Power Profiles) or a quoted string of values for the CUSTOM profile. - NOTE: These values can vary based on the ASIC, and may include: - -| Setting | Description | -|---------------------|----------------------------------------------------| -| SCLK_PROFILE_ENABLE | Whether or not to apply the 3 following SCLK settings (0=disable,1=enable) | -| | **NOTE: This is a hidden field. If set to 0, the following 3 values are displayed as '-’** | -| SCLK_UP_HYST | Delay before sclk is increased (in milliseconds) | -| SCLK_DOWN_HYST | Delay before sclk is decresed (in milliseconds) | -| SCLK_ACTIVE_LEVEL | Workload required before sclk levels change (in %) | -| MCLK_PROFILE_ENABLE | Whether or not to apply the 3 following MCLK settings (0=disable,1=enable) | -| | **NOTE: This is a hidden field. If set to 0, the following 3 values are displayed as '-'** | -| MCLK_UP_HYST | Delay before mclk is increased (in milliseconds) | -| MCLK_DOWN_HYST | Delay before mclk is decresed (in milliseconds) | -| MCLK_ACTIVE_LEVEL | Workload required before mclk levels change (in %) | - - Other settings: - -| Setting | Description | -|------------------|---------------------------------------------------------------------------| -| BUSY_SET_POINT | Threshold for raw activity level before levels change | -| FPS | Frames Per Second | -| USE_RLC_BUSY | When set to 1, DPM is switched up as long as RLC busy message is received | -| MIN_ACTIVE_LEVEL | Workload required before levels change (in %) | - - NOTES: - When a compute queue is detected, the COMPUTE Power Profile values will be automatically - applied to the system, provided that the Perf Level is set to "auto" - - The CUSTOM Power Profile is only applied when the Performance Level is set to "manual" - so using this flag will automatically set the performance level to "manual" - - It is not possible to modify the non-CUSTOM Profiles. These are hard-coded by the kernel - --P, --showpower: -Show average or instantaneous socket graphics package power consumption - -"Graphics Package" refers to the GPU plus any HBM (High-Bandwidth memory) modules, if present - --M, --showmaxpower: -Show the maximum Graphics Package power that the GPU will attempt to consume. -This limit is enforced by the hardware. - ---loglevel: -This will allow the user to set a logging level for the SMI's actions. Currently this is -only implemented for sysfs writes, but can easily be expanded upon in the future to log -other things from the SMI - ---showmeminfo: -This allows the user to see the amount of used and total memory for a given block (vram, -vis_vram, gtt). It returns the number of bytes used and total number of bytes for each block -'all' can be passed as a field to return all blocks, otherwise a quoted-string is used for -multiple values (e.g. "vram vis_vram") -vram refers to the Video RAM, or graphics memory, on the specified device -vis_vram refers to Visible VRAM, which is the CPU-accessible video memory on the device -gtt refers to the Graphics Translation Table - --b, --showbw: -This shows an approximation of the number of bytes received and sent by the GPU over -the last second through the PCIe bus. Note that this will not work for APUs since data for -the GPU portion of the APU goes through the memory fabric and does not 'enter/exit' -the chip via the PCIe interface, thus no accesses are generated, and the performance -counters can't count accesses that are not generated. -NOTE: It is not possible to easily grab the size of every packet that is transmitted -in real time, so the kernel estimates the bandwidth by taking the maximum payload size (mps), -which is the max size that a PCIe packet can be. and multiplies it by the number of packets -received and sent. This means that the SMI will report the maximum estimated bandwidth, -the actual usage could (and likely will be) less - ---showrasinfo: -This shows the RAS information for a given block. This includes enablement of the block -(currently GFX, SDMA and UMC are the only supported blocks) and the number of errors -ue - Uncorrectable errors -ce - Correctable errors - -## Clock Type Descriptions - -| Clock type | Description | -| ---------- | --- | -| DCEFCLK | DCE (Display) | -| FCLK | Data fabric (VG20 and later) - Data flow from XGMI, Memory, PCIe | -| SCLK | GFXCLK (Graphics core) | -| | **Note - SOCCLK split from SCLK as of Vega10. Pre-Vega10 they were both controlled by SCLK** | -| MCLK | GPU Memory (VRAM) | -| PCLK | PCIe bus | -| | **Note - This gives 2 speeds, PCIe Gen1 x1 and the highest available based on the hardware** | -| SOCCLK | System clock (VG10 and later) - Data Fabric (DF), MM HUB, AT HUB, SYSTEM HUB, OSS, DFD | -| | **Note - DF split from SOCCLK as of Vega20. Pre-Vega20 they were both controlled by SOCCLK** | - ---gpureset: -This flag will attempt to reset the GPU for a specified device. This will invoke the GPU reset through -the kernel debugfs file amdgpu_gpu_recover. Note that GPU reset will not always work, depending on the -manner in which the GPU is hung. - ---showdriverversion: -This flag will print out the AMDGPU module version for amdgpu-pro or ROCm kernels. For other kernels, -it will simply print out the name of the kernel (`uname -r`) - ---showserial: -This flag will print out the serial number for the graphics card - NOTE: This is currently only supported on Vega20 server cards that support it. Consumer cards and - cards older than Vega20 will not support this feature. - ---showproductname: -This uses the pci.ids file to print out more information regarding the GPUs on the system. -'update-pciids' may need to be executed on the machine to get the latest PCI ID snapshot, -as certain newer GPUs will not be present in the stock pci.ids file, and the file may even -be absent on certain OS installation types - ---showpagesinfo | --showretiredpages | --showpendingpages | --showunreservablepages: -These flags display the different "bad pages" as reported by the kernel. The three -types of pages are: -Retired pages (reserved pages) - These pages are reserved and are unable to be used -Pending pages - These pages are pending for reservation, and will be reserved/retired -Unreservable pages - These pages are not reservable for some reason - ---showmemuse | --showuse | --showmeminfo: ---showuse and --showmemuse are used to indicate how busy the respective blocks are. For -example, for --showuse (gpu_busy_percent sysfs file), the SMU samples every ms or so to see -if any GPU block (RLC, MEC, PFP, CP) is busy. If so, that's 1 (or high). If not, that's 0 (low). -If we have 5 high and 5 low samples, that means 50% utilization (50% GPU busy, or 50% GPU use). -The windows and sampling vary from generation to generation, but that is how GPU and VRAM use -is calculated in a generic sense. ---showmeminfo (and VRAM% in concise output) will show the amount of VRAM used (visible, total, GTT), -as well as the total available for those partitions. The percentage shown there indicates the -amount of used memory in terms of current allocations - -## OverDrive settings - -Enabling OverDrive requires both a card that support OverDrive and a driver parameter that enables its use. -Because OverDrive features can damage your card, most workstation and server GPUs cannot use OverDrive. -Consumer GPUs that can use OverDrive must enable this feature by setting bit 14 in the amdgpu driver's -ppfeaturemask module parameter - -For OverDrive functionality, the OverDrive bit (bit 14) must be enabled (by default, the -OverDrive bit is disabled on the ROCK and upstream kernels). This can be done by setting -amdgpu.ppfeaturemask accordingly in the kernel parameters, or by changing the default value -inside amdgpu_drv.c (if building your own kernel). - -As an example, if the ppfeaturemask is set to 0xffffbfff (11111111111111111011111111111111), -then enabling the OverDrive bit would make it 0xffffffff (11111111111111111111111111111111). - -These are the flags that require OverDrive functionality to be enabled for the flag to work: - --showclkvolt - --showvoltagerange - --showvc - --showsclkrange - --showmclkrange - --setslevel - --setmlevel - --setoverdrive - --setpoweroverdrive - --resetpoweroverdrive - --setvc - --setsrange - --setmrange - diff --git a/projects/rocm-smi-lib/docs/how-to/use-python.rst b/projects/rocm-smi-lib/docs/how-to/use-python.rst new file mode 100644 index 0000000000..f0ebd32493 --- /dev/null +++ b/projects/rocm-smi-lib/docs/how-to/use-python.rst @@ -0,0 +1,519 @@ +.. meta:: + :description: Learn about using the ROCm SMI library with Python. + :keywords: install, SMI, library, api, python, system management interface + +************************** +Using ROCm SMI with Python +************************** + +This tool acts as a command line interface for manipulating and monitoring the amdgpu kernel, and is intended to replace and deprecate the existing ``rocm_smi.py`` CLI tool. +It uses Ctypes to call the ``rocm_smi_lib`` API. + +To use ROCm SMI, you must have ROCm SMI library (``librocm_smi64``) installed and should have at least one AMD GPU with the ROCm driver installed. + +Installation +============ + +Follow the installation procedure for ``rocm_smi_lib``. Refer to :doc:`../install/install`. + +.. note:: + + ``hipcc`` and other compilers will not automatically link in the ``librocm_smi64`` dynamic library. To ensure the + ``librocm_smi64.so`` can be located, you must either set the ``LD_LIBRARY_PATH`` environment variable to the + directory containing ``librocm_smi64.so`` (usually ``/opt/rocm/lib`` or pass the ``-lrocm_smi64`` flag to the compiler. + +Version +======= + +The SMI will report two "versions": the ``ROCM-SMI`` version and the ``ROCM-SMI-LIB`` version. + +- ``ROCM-SMI`` version is the CLI/tool version number with commit ID appended after + sign. + +- ``ROCM-SMI-LIB`` version is the library package version number. + +.. code-block:: shell-session + + ROCM-SMI version: 2.0.0+8e78352 + ROCM-SMI-LIB version: 6.1.0 + +Usage +===== + +For detailed and up to date usage information, consult the help. + +.. code-block:: shell-session + + /opt/rocm/bin/rocm-smi -h + +The following is the output from the ``-h`` flag: + +.. code-block:: shell-session + + $ /opt/rocm/bin/rocm-smi -h + usage: rocm-smi [-h] [-V] [-d DEVICE [DEVICE ...]] [--alldevices] [--showhw] [-a] [-i] [-v] [-e [EVENT [EVENT ...]]] + [--showdriverversion] [--showtempgraph] [--showfwinfo [BLOCK [BLOCK ...]]] [--showmclkrange] + [--showmemvendor] [--showsclkrange] [--showproductname] [--showserial] [--showuniqueid] + [--showvoltagerange] [--showbus] [--showpagesinfo] [--showpendingpages] [--showretiredpages] + [--showunreservablepages] [-f] [-P] [-t] [-u] [--showmemuse] [--showvoltage] [-b] [-c] [-g] [-l] [-M] + [-m] [-o] [-p] [-S] [-s] [--showmeminfo TYPE [TYPE ...]] [--showpids [VERBOSE]] + [--showpidgpus [SHOWPIDGPUS [SHOWPIDGPUS ...]]] [--showreplaycount] + [--showrasinfo [SHOWRASINFO [SHOWRASINFO ...]]] [--showvc] [--showxgmierr] [--showtopo] + [--showtopoaccess] [--showtopoweight] [--showtopohops] [--showtopotype] [--showtoponuma] + [--showenergycounter] [--shownodesbw] [--showcomputepartition] [--showmemorypartition] [-r] + [--resetfans] [--resetprofile] [--resetpoweroverdrive] [--resetxgmierr] [--resetperfdeterminism] + [--resetcomputepartition] [--resetmemorypartition] [--setclock TYPE LEVEL] [--setsclk LEVEL [LEVEL ...]] + [--setmclk LEVEL [LEVEL ...]] [--setpcie LEVEL [LEVEL ...]] [--setslevel SCLKLEVEL SCLK SVOLT] + [--setmlevel MCLKLEVEL MCLK MVOLT] [--setvc POINT SCLK SVOLT] [--setsrange SCLKMIN SCLKMAX] + [--setextremum min|max sclk|mclk CLK] [--setmrange MCLKMIN MCLKMAX] [--setfan LEVEL] + [--setperflevel LEVEL] [--setoverdrive %] [--setmemoverdrive %] [--setpoweroverdrive WATTS] + [--setprofile SETPROFILE] [--setperfdeterminism SCLK] + [--setcomputepartition {CPX,SPX,DPX,TPX,QPX,cpx,spx,dpx,tpx,qpx}] + [--setmemorypartition {NPS1,NPS2,NPS4,NPS8,nps1,nps2,nps4,nps8}] [--rasenable BLOCK ERRTYPE] + [--rasdisable BLOCK ERRTYPE] [--rasinject BLOCK] [--gpureset] [--load FILE | --save FILE] + [--autorespond RESPONSE] [--loglevel LEVEL] [--json] [--csv] + + AMD ROCm System Management Interface | ROCM-SMI version: 2.0.0+8e78352 + + optional arguments: + -h, --help show this help message and exit + --gpureset Reset specified GPU (One GPU must be specified) + --load FILE Load Clock, Fan, Performance and Profile settings + from FILE + --save FILE Save Clock, Fan, Performance and Profile settings to + FILE + + -V, --version Show version information + + -d DEVICE [DEVICE ...], --device DEVICE [DEVICE ...] Execute command on specified device + + Display Options: + --alldevices + --showhw Show Hardware details + -a, --showallinfo Show Temperature, Fan and Clock values + + Topology: + -i, --showid Show DEVICE ID + -v, --showvbios Show VBIOS version + -e [EVENT [EVENT ...]], --showevents [EVENT [EVENT ...]] Show event list + --showdriverversion Show kernel driver version + --showtempgraph Show Temperature Graph + --showfwinfo [BLOCK [BLOCK ...]] Show FW information + --showmclkrange Show mclk range + --showmemvendor Show GPU memory vendor + --showsclkrange Show sclk range + --showproductname Show SKU/Vendor name + --showserial Show GPU's Serial Number + --showuniqueid Show GPU's Unique ID + --showvoltagerange Show voltage range + --showbus Show PCI bus number + + Pages information: + --showpagesinfo Show retired, pending and unreservable pages + --showpendingpages Show pending retired pages + --showretiredpages Show retired pages + --showunreservablepages Show unreservable pages + + Hardware-related information: + -f, --showfan Show current fan speed + -P, --showpower Show current Average or Socket Graphics Package Power + Consumption + -t, --showtemp Show current temperature + -u, --showuse Show current GPU use + --showmemuse Show current GPU memory used + --showvoltage Show current GPU voltage + + Software-related/controlled information: + -b, --showbw Show estimated PCIe use + -c, --showclocks Show current clock frequencies + -g, --showgpuclocks Show current GPU clock frequencies + -l, --showprofile Show Compute Profile attributes + -M, --showmaxpower Show maximum graphics package power this GPU will + consume + -m, --showmemoverdrive Show current GPU Memory Clock OverDrive level + -o, --showoverdrive Show current GPU Clock OverDrive level + -p, --showperflevel Show current DPM Performance Level + -S, --showclkvolt Show supported GPU and Memory Clocks and Voltages + -s, --showclkfrq Show supported GPU and Memory Clock + --showmeminfo TYPE [TYPE ...] Show Memory usage information for given block(s) TYPE + --showpids [VERBOSE] Show current running KFD PIDs (pass details to + VERBOSE for detailed information) + --showpidgpus [SHOWPIDGPUS [SHOWPIDGPUS ...]] Show GPUs used by specified KFD PIDs (all if no arg + given) + --showreplaycount Show PCIe Replay Count + --showrasinfo [SHOWRASINFO [SHOWRASINFO ...]] Show RAS enablement information and error counts for + the specified block(s) (all if no arg given) + --showvc Show voltage curve + --showxgmierr Show XGMI error information since last read + --showtopo Show hardware topology information + --showtopoaccess Shows the link accessibility between GPUs + --showtopoweight Shows the relative weight between GPUs + --showtopohops Shows the number of hops between GPUs + --showtopotype Shows the link type between GPUs + --showtoponuma Shows the numa nodes + --showenergycounter Energy accumulator that stores amount of energy + consumed + --shownodesbw Shows the numa nodes + --showcomputepartition Shows current compute partitioning + --showmemorypartition Shows current memory partition + + Set options: + --setclock TYPE LEVEL Set Clock Frequency Level(s) for specified clock + (requires manual Perf level) + --setsclk LEVEL [LEVEL ...] Set GPU Clock Frequency Level(s) (requires manual + Perf level) + --setmclk LEVEL [LEVEL ...] Set GPU Memory Clock Frequency Level(s) (requires + manual Perf level) + --setpcie LEVEL [LEVEL ...] Set PCIE Clock Frequency Level(s) (requires manual + Perf level) + --setslevel SCLKLEVEL SCLK SVOLT Change GPU Clock frequency (MHz) and Voltage (mV) for + a specific Level + --setmlevel MCLKLEVEL MCLK MVOLT Change GPU Memory clock frequency (MHz) and Voltage + for (mV) a specific Level + --setvc POINT SCLK SVOLT Change SCLK Voltage Curve (MHz mV) for a specific + point + --setsrange SCLKMIN SCLKMAX Set min and max SCLK speed + --setextremum min|max sclk|mclk CLK Set min/max of SCLK/MCLK speed + --setmrange MCLKMIN MCLKMAX Set min and max MCLK speed + --setfan LEVEL Set GPU Fan Speed (Level or %) + --setperflevel LEVEL Set Performance Level + --setoverdrive % Set GPU OverDrive level (requires manual|high Perf + level) + --setmemoverdrive % Set GPU Memory Overclock OverDrive level (requires + manual|high Perf level) + --setpoweroverdrive WATTS Set the maximum GPU power using Power OverDrive in + Watts + --setprofile SETPROFILE Specify Power Profile level (#) or a quoted string of + CUSTOM Profile attributes "# # # #..." (requires + manual Perf level) + --setperfdeterminism SCLK Set clock frequency limit to get minimal performance + variation + --setcomputepartition {CPX,SPX,DPX,TPX,QPX,cpx,spx,dpx,tpx,qpx} Set compute partition + --setmemorypartition {NPS1,NPS2,NPS4,NPS8,nps1,nps2,nps4,nps8} Set memory partition + --rasenable BLOCK ERRTYPE Enable RAS for specified block and error type + --rasdisable BLOCK ERRTYPE Disable RAS for specified block and error type + --rasinject BLOCK Inject RAS poison for specified block (ONLY WORKS ON + UNSECURE BOARDS) + + Reset options: + -r, --resetclocks Reset clocks and OverDrive to default + --resetfans Reset fans to automatic (driver) control + --resetprofile Reset Power Profile back to default + --resetpoweroverdrive Set the maximum GPU power back to the device deafult + state + --resetxgmierr Reset XGMI error count + --resetperfdeterminism Disable performance determinism + --resetcomputepartition Resets to boot compute partition state + --resetmemorypartition Resets to boot memory partition state + + Auto-response options: + --autorespond RESPONSE Response to automatically provide for all prompts + (NOT RECOMMENDED) + + Output options: + --loglevel LEVEL How much output will be printed for what program is + doing, one of debug/info/warning/error/critical + --json Print output in JSON format + --csv Print output in CSV format + +Detailed option descriptions +============================ + +--setextremum <[min or max] [sclk or mclk] [value in MHz to set to]> + Provided ASIC support, users can now set a maximum or minimum sclk or mclk value through our Python CLI tool (`rocm-smi --setextremum max sclk 1500`). See example below. + + .. code-block:: shell-session + + $ sudo /opt/rocm/bin/rocm-smi --setextremum max sclk 2100 + + ============================ ROCm System Management Interface ============================ + + ******WARNING****** + + Operating your AMD GPU outside of official AMD specifications or outside of + factory settings, including but not limited to the conducting of overclocking, + over-volting or under-volting (including use of this interface software, + even if such software has been directly or indirectly provided by AMD or otherwise + affiliated in any way with AMD), may cause damage to your AMD GPU, system components + and/or result in system failure, as well as cause other problems. + DAMAGES CAUSED BY USE OF YOUR AMD GPU OUTSIDE OF OFFICIAL AMD SPECIFICATIONS OR + OUTSIDE OF FACTORY SETTINGS ARE NOT COVERED UNDER ANY AMD PRODUCT WARRANTY AND + MAY NOT BE COVERED BY YOUR BOARD OR SYSTEM MANUFACTURER'S WARRANTY. + Use this utility with caution. + + Do you accept these terms? [y/N] y + ================================ Set Valid sclk Extremum ================================= + GPU[0] : Successfully set max sclk to 2100(MHz) + GPU[1] : Successfully set max sclk to 2100(MHz) + GPU[2] : Successfully set max sclk to 2100(MHz) + GPU[3] : Successfully set max sclk to 2100(MHz) + ================================== End of ROCm SMI Log =================================== + +--setsclk, --setmclk <# [# # ...]> + This allows you to set a mask for the levels. For example, if a GPU has 8 clock levels, + you can set a mask to use levels 0, 5, 6 and 7 with --setsclk 0 5 6 7 . This will only + use the base level, and the top 3 clock levels. This will allow you to keep the GPU at + base level when there is no GPU load, and the top 3 levels when the GPU load increases. + + .. note:: + + The clock levels will change dynamically based on GPU load based on the default + Compute and Graphics profiles. The thresholds and delays for a custom mask cannot + be controlled through the SMI tool. + + This flag automatically sets the Performance Level to ``manual`` as the mask is not + applied when the Performance level is set to ``auto``. + +--setfan + This sets the fan speed to a value ranging from 0 to maxlevel, or from 0%-100%. + + If the level ends with a %, the fan speed is calculated as pct*maxlevel/100 (maxlevel is usually 255, but is determined by the ASIC). + + .. note:: + + While the hardware is usually capable of overriding this value when required, it is + recommended to not set the fan level lower than the default value for extended periods + of time. + +--setperflevel + This lets you use the pre-defined Performance Level values for clocks and power profile, which can include: + + * ``auto`` (Automatically change values based on GPU workload) + + * ``low`` (Keep values low, regardless of workload) + + * ``high`` (Keep values high, regardless of workload) + + * ``manual`` (Only use values defined by ``--setsclk`` and ``--setmclk``) + +--setoverdrive, --setmemoverdrive <#> + .. warning:: + + DEPRECATED IN NEWER KERNEL VERSIONS. Use ``--setslevel`` or ``--setmlevel`` instead. + + This sets the percentage above maximum for the max Performance Level. + For example, ``--setoverdrive`` 20 will increase the top sclk level by 20%, similarly + ``--setmemoverdrive`` 20 will increase the top mclk level by 20%. Thus if the maximum + clock level is 1000MHz, then ``--setoverdrive`` 20 will increase the maximum clock to 1200MHz. + + .. note:: + + This option can be used in conjunction with the ``--setsclk``/``--setmclk`` mask. + + Operating the GPU outside of specifications can cause irreparable damage to your hardware + Observe the warning displayed when using this option. + + This flag automatically sets the clock to the highest level, as only the highest level is + increased by the :ref:`OverDrive ` value. + +--setpoweroverdrive, --resetpoweroverdrive <#> + This allows users to change the maximum power available to a GPU package. + The input value is in Watts. This limit is enforced by the hardware, and + some cards allow users to set it to a higher value than the default that + ships with the GPU. This Power OverDrive mode allows the GPU to run at + higher frequencies for longer periods of time, though this may mean the + GPU uses more power than it is allowed to use per power supply + specifications. Each GPU has a model-specific maximum Power OverDrive that + is will take; attempting to set a higher limit than that will cause this + command to fail. + + .. note:: + + Operating the GPU outside of specifications can cause irreparable damage to your hardware. + Observe the warning displayed when using this option. + +--setprofile + The Compute Profile accepts 1 or n parameters, either the Profile to select (see ``--showprofile`` for a list + of preset Power Profiles) or a quoted string of values for the CUSTOM profile. + These values can vary based on the ASIC, and may include: + + .. code-block:: shell-session + + | Setting | Description | + |---------------------|--------------------------------------------------------------------------------------------| + | SCLK_PROFILE_ENABLE | Whether or not to apply the 3 following SCLK settings (0=disable,1=enable) | + | | **NOTE: This is a hidden field. If set to 0, the following 3 values are displayed as '-’** | + | SCLK_UP_HYST | Delay before sclk is increased (in milliseconds) | + | SCLK_DOWN_HYST | Delay before sclk is decresed (in milliseconds) | + | SCLK_ACTIVE_LEVEL | Workload required before sclk levels change (in %) | + | MCLK_PROFILE_ENABLE | Whether or not to apply the 3 following MCLK settings (0=disable,1=enable) | + | | **NOTE: This is a hidden field. If set to 0, the following 3 values are displayed as '-'** | + | MCLK_UP_HYST | Delay before mclk is increased (in milliseconds) | + | MCLK_DOWN_HYST | Delay before mclk is decresed (in milliseconds) | + | MCLK_ACTIVE_LEVEL | Workload required before mclk levels change (in %) | + +Other settings +-------------- + +.. code-block:: shell-session + + | Setting | Description | + |------------------|---------------------------------------------------------------------------| + | BUSY_SET_POINT | Threshold for raw activity level before levels change | + | FPS | Frames Per Second | + | USE_RLC_BUSY | When set to 1, DPM is switched up as long as RLC busy message is received | + | MIN_ACTIVE_LEVEL | Workload required before levels change (in %) | + +.. note:: + + When a compute queue is detected, the COMPUTE Power Profile values will be automatically + applied to the system, provided that the Perf Level is set to ``auto``. + + The CUSTOM Power Profile is only applied when the Performance Level is set to ``manual`` + so using this flag will automatically set the performance level to ``manual``. + + It is not possible to modify the non-CUSTOM Profiles. These are hard-coded by the kernel. + +-P, --showpower + Show average or instantaneous socket graphics package power consumption. + + "Graphics Package" refers to the GPU plus any HBM (High-Bandwidth memory) modules, if present. + +-M, --showmaxpower + Show the maximum Graphics Package power that the GPU will attempt to consume. + + This limit is enforced by the hardware. + +--loglevel + This will allow the user to set a logging level for the SMI's actions. Currently this is + only implemented for ``sysfs`` writes, but can easily be expanded upon in the future to log + other things from the SMI. + +--showmeminfo + This allows the user to see the amount of used and total memory for a given block (``vram``, + ``vis_vram``, ``gtt``). It returns the number of bytes used and total number of bytes for each block + ``all`` can be passed as a field to return all blocks, otherwise a quoted-string is used for + multiple values (such as ``vram vis_vram``). + + ``vram`` refers to the Video RAM, or graphics memory, on the specified device + ``vis_vram`` refers to Visible VRAM, which is the CPU-accessible video memory on the device + ``gtt`` refers to the Graphics Translation Table + +-b, --showbw + This shows an approximation of the number of bytes received and sent by the GPU over + the last second through the PCIe bus. Note that this will not work for APUs since data for + the GPU portion of the APU goes through the memory fabric and does not 'enter/exit' + the chip via the PCIe interface, thus no accesses are generated, and the performance + counters can't count accesses that are not generated. + + .. note:: + + It is not possible to easily grab the size of every packet that is transmitted + in real time, so the kernel estimates the bandwidth by taking the maximum payload size (mps), + which is the max size that a PCIe packet can be. and multiplies it by the number of packets + received and sent. This means that the SMI will report the maximum estimated bandwidth, + the actual usage could (and likely will be) less. + +--showrasinfo + This shows the RAS information for a given block. This includes enablement of the block + (currently GFX, SDMA and UMC are the only supported blocks) and the number of errors. + + * ``ue`` - Uncorrectable errors + + * ``ce`` - Correctable errors + +Clock type descriptions +======================= + +.. code-block:: shell-session + + | Clock type | Description | + | ---------- | -------------------------------------------------------------------------------------------- | + | DCEFCLK | DCE (Display) | + | FCLK | Data fabric (VG20 and later) - Data flow from XGMI, Memory, PCIe | + | SCLK | GFXCLK (Graphics core) | + | | **Note - SOCCLK split from SCLK as of Vega10. Pre-Vega10 they were both controlled by SCLK** | + | MCLK | GPU Memory (VRAM) | + | PCLK | PCIe bus | + | | **Note - This gives 2 speeds, PCIe Gen1 x1 and the highest available based on the hardware** | + | SOCCLK | System clock (VG10 and later) - Data Fabric (DF), MM HUB, AT HUB, SYSTEM HUB, OSS, DFD | + | | **Note - DF split from SOCCLK as of Vega20. Pre-Vega20 they were both controlled by SOCCLK** | + +--gpureset + This flag will attempt to reset the GPU for a specified device. This will invoke the GPU reset through + the kernel debugfs file ``amdgpu_gpu_recover``. Note that GPU reset will not always work, depending on the + manner in which the GPU is hung. + +--showdriverversion + This flag will print out the AMDGPU module version for amdgpu-pro or ROCm kernels. For other kernels, + it will simply print out the name of the kernel (``uname -r``). + +--showserial + This flag will print out the serial number for the graphics card. + + .. note:: + + This is currently only supported on Vega20 server cards that support it. Consumer cards and + cards older than Vega20 will not support this feature. + +--showproductname + This uses the ``pci.ids`` file to print out more information regarding the GPUs on the system. + ``update-pciids`` may need to be executed on the machine to get the latest PCI ID snapshot, + as certain newer GPUs will not be present in the stock ``pci.ids`` file, and the file may even + be absent on certain OS installation types. + +--showpagesinfo, --showretiredpages, --showpendingpages, --showunreservablepages + These flags display the different "bad pages" as reported by the kernel. The three + types of pages are: + + * Retired pages (reserved pages) - These pages are reserved and are unable to be used. + + * Pending pages - These pages are pending for reservation, and will be reserved/retired. + + * Unreservable pages - These pages are not reservable for some reason. + +--showmemuse, --showuse, --showmeminfo + ``--showuse`` and ``--showmemuse`` are used to indicate how busy the respective blocks are. For + example, for ``--showuse (gpu_busy_percent sysfs file)``, the SMU samples every ms or so to see + if any GPU block (RLC, MEC, PFP, CP) is busy. If so, that's 1 (or high). If not, that's 0 (low). + If we have 5 high and 5 low samples, that means 50% utilization (50% GPU busy, or 50% GPU use). + The windows and sampling vary from generation to generation, but that is how GPU and VRAM use + is calculated in a generic sense. + ``--showmeminfo`` (and VRAM% in concise output) will show the amount of VRAM used (visible, total, GTT), + as well as the total available for those partitions. The percentage shown there indicates the + amount of used memory in terms of current allocations. + +.. _overdrive-settings: + +OverDrive settings +================== + +Enabling OverDrive requires both a card that support OverDrive and a driver parameter that enables its use. +Because OverDrive features can damage your card, most workstation and server GPUs cannot use OverDrive. +Consumer GPUs that can use OverDrive must enable this feature by setting bit 14 in the amdgpu driver's +ppfeaturemask module parameter + +For OverDrive functionality, the OverDrive bit (bit 14) must be enabled (by default, the +OverDrive bit is disabled on the ROCK and upstream kernels). This can be done by setting +amdgpu.ppfeaturemask accordingly in the kernel parameters, or by changing the default value +inside amdgpu_drv.c (if building your own kernel). + +As an example, if the ``ppfeaturemask`` is set to ``0xffffbfff`` (``11111111111111111011111111111111``), +then enabling the OverDrive bit would make it ``0xffffffff`` (``11111111111111111111111111111111``). + +These are the flags that require OverDrive functionality to be enabled for the flag to work: + +* ``--showclkvolt`` + +* ``--showvoltagerange`` + +* ``--showvc`` + +* ``--showsclkrange`` + +* ``--showmclkrange`` + +* ``--setslevel`` + +* ``--setmlevel`` + +* ``--setoverdrive`` + +* ``--setpoweroverdrive`` + +* ``--resetpoweroverdrive`` + +* ``--setvc`` + +* ``--setsrange`` + +* ``--setmrange`` diff --git a/projects/rocm-smi-lib/docs/index.rst b/projects/rocm-smi-lib/docs/index.rst index 8e531214d2..a92c802e3a 100755 --- a/projects/rocm-smi-lib/docs/index.rst +++ b/projects/rocm-smi-lib/docs/index.rst @@ -6,17 +6,19 @@ ROCm System Management Interface (ROCm SMI) library **************************************************** -The ROCm System Management Interface library, or ROCm SMI library, is part of the ROCm software stack. It is a C library for Linux that provides a user space interface for applications to monitor and control GPU applications. +The ROCm SMI library, is part of the ROCm software stack. It is a C++ library for Linux that provides a user space interface for applications to monitor and control GPU applications. -For more information, refer to `GitHub. `_ +For more information, refer to ``__. -.. NOTE:: +.. note:: -The AMD System Management Interface Library (AMD SMI library) is a C library for Linux that provides a user space interface for applications to monitor and control AMD devices. This library will replace rocm_smi_lib over time. We recommend that users transition to the AMD SMI library. - -For more information, refer to the `GitHub repository `_. + The AMD System Management Interface Library (AMD SMI) is a successor to ROCm SMI. It is a unified system management + interface tool that provides a user space interface for applications to monitor and control GPU applications and gives + users the ability to query information about drivers and GPUs on the system. + AMD SMI will replace ``rocm_smi_lib`` over time. We recommend that users transition to AMD SMI. + For more information, refer to ``__ and the :doc:`AMD SMI documentation `. .. grid:: 2 :gutter: 3 diff --git a/projects/rocm-smi-lib/docs/sphinx/_toc.yml.in b/projects/rocm-smi-lib/docs/sphinx/_toc.yml.in index 770e4caa3b..804d949b14 100644 --- a/projects/rocm-smi-lib/docs/sphinx/_toc.yml.in +++ b/projects/rocm-smi-lib/docs/sphinx/_toc.yml.in @@ -14,9 +14,7 @@ subtrees: - caption: How to entries: - file: how-to/use-cpp - title: Use C++ in ROCm SMI - file: how-to/use-python - title: Use Python in ROCm SMI - caption: API Reference entries: From 0b40a737987920a6b21021e58a1eddd02906bb61 Mon Sep 17 00:00:00 2001 From: Charis Poag Date: Fri, 21 Jun 2024 15:13:15 -0500 Subject: [PATCH 2/2] [SWDEV-422195/SWDEV-440985] GPU metrics 1.6 + --showmetrics Changes: - Added new GPU metrics: 1) Violation status' (ex. PVIOL/TVIOL) accumulators 2) XCP (Graphics Compute Partitions) statistics 3) pcie other end recovery counter - Added rocm-smi --showmetrics Units/values reflect as indicated by driver, may differ from AMD SMI or other ROCm SMI interfaces which use these fields. - N/A fields means the device does not support providing this data. Change-Id: Ia2cd3bb65c4f474ebdb39db8062ea716f2b4d8ee Signed-off-by: Charis Poag [ROCm/rocm_smi_lib commit: 0609cbf1d0dcf0057004254442ba646b33bf1820] --- projects/rocm-smi-lib/CHANGELOG.md | 71 +- .../rocm-smi-lib/include/rocm_smi/rocm_smi.h | 112 +- .../include/rocm_smi/rocm_smi_device.h | 4 + .../include/rocm_smi/rocm_smi_gpu_metrics.h | 367 +++-- .../include/rocm_smi/rocm_smi_utils.h | 72 + .../rocm-smi-lib/python_smi_tools/rocm_smi.py | 330 ++++- .../python_smi_tools/rsmiBindings.py | 99 ++ .../rocm_smi/example/rocm_smi_example.cc | 120 +- projects/rocm-smi-lib/src/rocm_smi.cc | 3 + projects/rocm-smi-lib/src/rocm_smi_device.cc | 16 +- .../rocm-smi-lib/src/rocm_smi_gpu_metrics.cc | 1253 +++++++++++++---- .../functional/gpu_metrics_read.cc | 343 +++-- .../functional/measure_api_execution_time.cc | 120 +- 13 files changed, 2308 insertions(+), 602 deletions(-) diff --git a/projects/rocm-smi-lib/CHANGELOG.md b/projects/rocm-smi-lib/CHANGELOG.md index d0c8d74dfc..56097897a8 100644 --- a/projects/rocm-smi-lib/CHANGELOG.md +++ b/projects/rocm-smi-lib/CHANGELOG.md @@ -4,22 +4,68 @@ Full documentation for rocm_smi_lib is available at [https://rocm.docs.amd.com/] ***All information listed below is for reference and subject to change.*** + +## rocm_smi_lib for ROCm 6.3 + +### Changes + +- **Added support for GPU metrics 1.6 to `rsmi_dev_gpu_metrics_info_get()`** +Updated `rsmi_dev_gpu_metrics_info_get()` and structure `rsmi_gpu_metrics_t` to include new fields for PVIOL / TVIOL, XCP (Graphics Compute Partitions) stats, and pcie_lc_perf_other_end_recovery: + - `uint64_t accumulation_counter` - used for all throttled calculations + - `uint64_t prochot_residency_acc` - Processor hot accumulator + - `uint64_t ppt_residency_acc` - Package Power Tracking (PPT) accumulator (used in PVIOL calculations) + - `uint64_t socket_thm_residency_acc` - Socket thermal accumulator - (used in TVIOL calculations) + - `uint64_t vr_thm_residency_acc` - Voltage Rail (VR) thermal accumulator + - `uint64_t hbm_thm_residency_acc` - High Bandwidth Memory (HBM) thermal accumulator + - `uint16_t num_partition` - corresponds to the current total number of partitions + - `struct amdgpu_xcp_metrics_t xcp_stats[MAX_NUM_XCP]` - for each partition associated with current GPU, provides gfx busy & accumulators, jpeg, and decoder (VCN) engine utilizations + - `uint32_t gfx_busy_inst[MAX_NUM_XCC]` - graphic engine utilization (%) + - `uint16_t jpeg_busy[MAX_NUM_JPEG_ENGS]` - jpeg engine utilization (%) + - `uint16_t vcn_busy[MAX_NUM_VCNS]` - decoder (VCN) engine utilization (%) + - `uint64_t gfx_busy_acc[MAX_NUM_XCC]` - graphic engine utilization accumulated (%) + - `uint32_t pcie_lc_perf_other_end_recovery` - corresponds to the pcie other end recovery counter + +- **Added ability to view raw GPU metrics`rocm-smi --showmetrics`** +Users can now view GPU metrics from our new `rocm-smi --showmetrics`. Unlike AMD SMI (or other ROCM-SMI interfaces), these values are ***not*** converted into applicable units as users may see in `amd-smi metric`. Units listed display as indicated by the driver, they are not converted (eg. in other AMD SMI/ROCm SMI interfaces which use the data provided). It is important to note, that fields displaying `N/A` data mean this ASIC does not support or backward compatibility was not provided in a newer ASIC's GPU metric structure. + +### Removals + +- N/A + +### Optimizations + +- N/A + +### Resolved issues + +- N/A + + +### Known Issues + +- N/A + +### Upcoming changes + +- N/A + + ## rocm_smi_lib for ROCm 6.2.1 -### Added +### Changes - N/A -### Changed +### Removals - N/A -### Optimized +### Optimizations - **Improved handling of UnicodeEncodeErrors with non UTF-8 locales** Non UTF-8 locales were causing crashing on UTF-8 special characters -### Fixed +### Resolved issues - **Fixed rsmitstReadWrite.TestComputePartitionReadWrite segfault** Segfault was caused due to unhandled start conditions: @@ -36,28 +82,33 @@ c. reload amgpu - `sudo modprobe amdgpu` Test needed to keep track of total number of devices, in order to ensure test comes back to the original configuration. The test segfault could be seen on all MI3x ASICs, if brought up in a non-SPX configuration upon boot. + ### Known Issues - N/A +### Upcoming changes + +- N/A + ## rocm_smi_lib for ROCm 6.2 -### Added +### Changes - **Added Partition ID API (`rsmi_dev_partition_id_get(..)`)** Previously `rsmi_dev_partition_id_get` could only be retrived by querying through `rsmi_dev_pci_id_get()` and parsing optional bits in our python CLI/API. We are now making this available directly through API. As well as added testing, in our compute partitioning tests verifing partition IDs update accordingly. -### Changed +### Removals - N/A -### Optimized +### Optimizations - N/A -### Fixed +### Resolved issues - **Partition ID CLI output** Due to driver changes in KFD, some devices may report bits [31:28] or [2:0]. With the newly added `rsmi_dev_partition_id_get(..)`, we provided this fallback to properly retreive partition ID. We @@ -74,6 +125,10 @@ plan to eventually remove partition ID from the function portion of the BDF (Bus - N/A +### Upcoming changes + +- N/A + ## rocm_smi_lib for ROCm 6.1.2 ### Added diff --git a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h index 7ed96aaf09..bd56f887ee 100755 --- a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h +++ b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h @@ -925,10 +925,6 @@ struct metrics_table_header_t { typedef struct metrics_table_header_t metrics_table_header_t; /// \endcond -/** - * @brief The following structure holds the gpu metrics values for a device. - */ - /** * @brief Unit conversion factor for HBM temperatures */ @@ -964,6 +960,41 @@ typedef struct metrics_table_header_t metrics_table_header_t; */ #define RSMI_MAX_NUM_GFX_CLKS 8 +/** + * @brief This should match kRSMI_MAX_NUM_XCC; + * XCC - Accelerated Compute Core, the collection of compute units, + * ACE (Asynchronous Compute Engines), caches, + * and global resources organized as one unit. + * + * Refer to amd.com documentation for more detail: + * https://www.amd.com/content/dam/amd/en/documents/instinct-tech-docs/white-papers/amd-cdna-3-white-paper.pdf + */ +#define RSMI_MAX_NUM_XCC 8 + +/** + * @brief This should match kRSMI_MAX_NUM_XCP; + * XCP - Accelerated Compute Processor, + * also referred to as the Graphics Compute Partitions. + * Each physical gpu could have a maximum of 8 separate partitions + * associated with each (depending on ASIC support). + * + * Refer to amd.com documentation for more detail: + * https://www.amd.com/content/dam/amd/en/documents/instinct-tech-docs/white-papers/amd-cdna-3-white-paper.pdf + */ +#define RSMI_MAX_NUM_XCP 8 + +/** + * @brief The following structures hold the gpu statistics for a device. + */ +struct amdgpu_xcp_metrics_t { + /* Utilization Instantaneous (%) */ + uint32_t gfx_busy_inst[RSMI_MAX_NUM_XCC]; + uint16_t jpeg_busy[RSMI_MAX_NUM_JPEG_ENGS]; + uint16_t vcn_busy[RSMI_MAX_NUM_VCNS]; + + /* Utilization Accumulated (%) */ + uint64_t gfx_busy_acc[RSMI_MAX_NUM_XCC]; +}; typedef struct { // TODO(amd) Doxygen documents @@ -985,7 +1016,7 @@ typedef struct { */ struct metrics_table_header_t common_header; - // Temperature + // Temperature (C) uint16_t temperature_edge; uint16_t temperature_hotspot; uint16_t temperature_mem; @@ -993,19 +1024,19 @@ typedef struct { uint16_t temperature_vrsoc; uint16_t temperature_vrmem; - // Utilization + // Utilization (%) uint16_t average_gfx_activity; uint16_t average_umc_activity; // memory controller uint16_t average_mm_activity; // UVD or VCN - // Power/Energy + // Power (W) /Energy (15.259uJ per 1ns) uint16_t average_socket_power; uint64_t energy_accumulator; // v1 mod. (32->64) // Driver attached timestamp (in ns) uint64_t system_clock_counter; // v1 mod. (moved from top of struct) - // Average clocks + // Average clocks (MHz) uint16_t average_gfxclk_frequency; uint16_t average_socclk_frequency; uint16_t average_uclk_frequency; @@ -1014,7 +1045,7 @@ typedef struct { uint16_t average_vclk1_frequency; uint16_t average_dclk1_frequency; - // Current clocks + // Current clocks (MHz) uint16_t current_gfxclk; uint16_t current_socclk; uint16_t current_uclk; @@ -1026,10 +1057,10 @@ typedef struct { // Throttle status uint32_t throttle_status; - // Fans + // Fans (RPM) uint16_t current_fan_speed; - // Link width/speed + // Link width (number of lanes) /speed (0.1 GT/s) uint16_t pcie_link_width; // v1 mod.(8->16) uint16_t pcie_link_speed; // in 0.1 GT/s; v1 mod. (8->16) @@ -1045,7 +1076,7 @@ typedef struct { /* * v1.2 additions */ - // PMFW attached timestamp (10ns resolution) + // PMFW attached timestamp (10ns resolution) uint64_t firmware_timestamp; @@ -1068,19 +1099,19 @@ typedef struct { uint16_t current_socket_power; // Utilization (%) - uint16_t vcn_activity[RSMI_MAX_NUM_VCNS]; // VCN instances activity percent (encode/decode) + uint16_t vcn_activity[RSMI_MAX_NUM_VCNS]; // VCN instances activity percent (encode/decode) // Clock Lock Status. Each bit corresponds to clock instance uint32_t gfxclk_lock_status; - // XGMI bus width and bitrate (in Gbps) + // XGMI bus width and bitrate (in GB/s) uint16_t xgmi_link_width; uint16_t xgmi_link_speed; // PCIE accumulated bandwidth (GB/sec) uint64_t pcie_bandwidth_acc; - // PCIE instantaneous bandwidth (GB/sec) + // PCIE instantaneous bandwidth (GB/sec) uint64_t pcie_bandwidth_inst; // PCIE L0 to recovery state transition accumulated count @@ -1114,6 +1145,57 @@ typedef struct { // PCIE NAK received accumulated count uint32_t pcie_nak_rcvd_count_acc; + /* + * v1.6 additions + */ + /* Accumulation cycle counter */ + uint64_t accumulation_counter; + + /** + * Accumulated throttler residencies + */ + uint64_t prochot_residency_acc; + /** + * Accumulated throttler residencies + * + * Prochot (thermal) - PPT (power) + * Package Power Tracking (PPT) violation % (greater than 0% is a violation); + * aka PVIOL + * + * Ex. PVIOL/TVIOL calculations + * Where A and B are measurments recorded at prior points in time. + * Typically A is the earlier measured value and B is the latest measured value. + * + * PVIOL % = (PptResidencyAcc (B) - PptResidencyAcc (A)) * 100/ (AccumulationCounter (B) - AccumulationCounter (A)) + * TVIOL % = (SocketThmResidencyAcc (B) - SocketThmResidencyAcc (A)) * 100 / (AccumulationCounter (B) - AccumulationCounter (A)) + */ + uint64_t ppt_residency_acc; + /** + * Accumulated throttler residencies + * + * Socket (thermal) - + * Socket thermal violation % (greater than 0% is a violation); + * aka TVIOL + * + * Ex. PVIOL/TVIOL calculations + * Where A and B are measurments recorded at prior points in time. + * Typically A is the earlier measured value and B is the latest measured value. + * + * PVIOL % = (PptResidencyAcc (B) - PptResidencyAcc (A)) * 100/ (AccumulationCounter (B) - AccumulationCounter (A)) + * TVIOL % = (SocketThmResidencyAcc (B) - SocketThmResidencyAcc (A)) * 100 / (AccumulationCounter (B) - AccumulationCounter (A)) + */ + uint64_t socket_thm_residency_acc; + uint64_t vr_thm_residency_acc; + uint64_t hbm_thm_residency_acc; + + /* Number of current partition */ + uint16_t num_partition; + + /* XCP (Graphic Cluster Partitions) metrics stats */ + struct amdgpu_xcp_metrics_t xcp_stats[RSMI_MAX_NUM_XCP]; + + /* PCIE other end recovery counter */ + uint32_t pcie_lc_perf_other_end_recovery; /// \endcond } rsmi_gpu_metrics_t; diff --git a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_device.h b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_device.h index 721515e456..0101a2c20a 100755 --- a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_device.h +++ b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_device.h @@ -242,6 +242,8 @@ class Device { AMGpuMetricsPublicLatestTupl_t dev_copy_internal_to_external_metrics(); static const std::map devInfoTypesStrings; + void set_smi_device_id(uint32_t i) { m_device_id = i; } + void set_smi_partition_id(uint32_t i) { m_partition_id = i; } private: std::shared_ptr monitor_; @@ -278,6 +280,8 @@ class Device { GpuMetricsBasePtr m_gpu_metrics_ptr; AMDGpuMetricsHeader_v1_t m_gpu_metrics_header; uint64_t m_gpu_metrics_updated_timestamp; + uint32_t m_device_id; + uint32_t m_partition_id; }; diff --git a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_gpu_metrics.h b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_gpu_metrics.h index b6cccdc6a2..00c0f8e70b 100644 --- a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_gpu_metrics.h +++ b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_gpu_metrics.h @@ -3,7 +3,7 @@ * The University of Illinois/NCSA * Open Source License (NCSA) * - * Copyright (c) 2017-2023, Advanced Micro Devices, Inc. + * Copyright (c) 2017-2024, Advanced Micro Devices, Inc. * All rights reserved. * * Developed by: @@ -52,6 +52,7 @@ #include #include #include +#include #include #include #include @@ -64,21 +65,19 @@ * All 1.4 and newer GPU metrics are now defined in this header. * */ -namespace amd::smi -{ +namespace amd::smi { constexpr uint32_t kRSMI_GPU_METRICS_API_CONTENT_MAJOR_VER_1 = 1; constexpr uint32_t kRSMI_GPU_METRICS_API_CONTENT_MINOR_VER_1 = 1; constexpr uint32_t kRSMI_GPU_METRICS_API_CONTENT_MINOR_VER_2 = 2; constexpr uint32_t kRSMI_GPU_METRICS_API_CONTENT_MINOR_VER_3 = 3; constexpr uint32_t kRSMI_GPU_METRICS_API_CONTENT_MINOR_VER_4 = 4; -constexpr uint32_t kRSMI_LATEST_GPU_METRICS_API_CONTENT_MAJOR_VER = kRSMI_GPU_METRICS_API_CONTENT_MAJOR_VER_1; -constexpr uint32_t kRSMI_LATEST_GPU_METRICS_API_CONTENT_MINON_VER = kRSMI_GPU_METRICS_API_CONTENT_MINOR_VER_4; +constexpr uint32_t kRSMI_LATEST_GPU_METRICS_API_CONTENT_MAJOR_VER + = kRSMI_GPU_METRICS_API_CONTENT_MAJOR_VER_1; +constexpr uint32_t kRSMI_LATEST_GPU_METRICS_API_CONTENT_MINON_VER + = kRSMI_GPU_METRICS_API_CONTENT_MINOR_VER_4; -// Note: As gpu metrics are updating -constexpr uint32_t kRSMI_GPU_METRICS_EXPIRATION_SECS = 5; - // Note: This *must* match NUM_HBM_INSTANCES constexpr uint32_t kRSMI_MAX_NUM_HBM_INSTANCES = 4; @@ -97,23 +96,36 @@ constexpr uint32_t kRSMI_MAX_NUM_VCNS = 4; // Note: This *must* match NUM_JPEG_ENG constexpr uint32_t kRSMI_MAX_JPEG_ENGINES = 32; +// Note: This *must* match MAX_XCC +constexpr uint32_t kRSMI_MAX_NUM_XCC = 8; -struct AMDGpuMetricsHeader_v1_t -{ +// Note: This *must* match MAX_XCP +constexpr uint32_t kRSMI_MAX_NUM_XCP = 8; + + +struct AMDGpuMetricsHeader_v1_t { uint16_t m_structure_size; uint8_t m_format_revision; uint8_t m_content_revision; }; -struct AMDGpuMetricsBase_t -{ +struct amdgpu_xcp_metrics { + /* Utilization Instantaneous (%) */ + uint32_t gfx_busy_inst[kRSMI_MAX_NUM_XCC]; + uint16_t jpeg_busy[kRSMI_MAX_JPEG_ENGINES]; + uint16_t vcn_busy[kRSMI_MAX_NUM_VCNS]; + + /* Utilization Accumulated (%) */ + uint64_t gfx_busy_acc[kRSMI_MAX_NUM_XCC]; +}; + +struct AMDGpuMetricsBase_t { virtual ~AMDGpuMetricsBase_t() = default; }; using AMDGpuMetricsBaseRef = AMDGpuMetricsBase_t&; -struct AMDGpuMetrics_v11_t -{ +struct AMDGpuMetrics_v11_t { ~AMDGpuMetrics_v11_t() = default; struct AMDGpuMetricsHeader_v1_t m_common_header; @@ -174,8 +186,7 @@ struct AMDGpuMetrics_v11_t uint16_t m_temperature_hbm[kRSMI_MAX_NUM_HBM_INSTANCES]; }; -struct AMDGpuMetrics_v12_t -{ +struct AMDGpuMetrics_v12_t { ~AMDGpuMetrics_v12_t() = default; struct AMDGpuMetricsHeader_v1_t m_common_header; @@ -238,8 +249,7 @@ struct AMDGpuMetrics_v12_t uint64_t m_firmware_timestamp; }; -struct AMDGpuMetrics_v13_t -{ +struct AMDGpuMetrics_v13_t { ~AMDGpuMetrics_v13_t() = default; struct AMDGpuMetricsHeader_v1_t m_common_header; @@ -298,7 +308,7 @@ struct AMDGpuMetrics_v13_t uint32_t m_mem_activity_acc; // new in v1 uint16_t m_temperature_hbm[kRSMI_MAX_NUM_HBM_INSTANCES]; // new in v1 - // PMFW attached timestamp (10ns resolution) + // PMFW attached timestamp (10ns resolution) uint64_t m_firmware_timestamp; // Voltage (mV) @@ -312,8 +322,7 @@ struct AMDGpuMetrics_v13_t uint64_t m_indep_throttle_status; }; -struct AMDGpuMetrics_v14_t -{ +struct AMDGpuMetrics_v14_t { ~AMDGpuMetrics_v14_t() = default; struct AMDGpuMetricsHeader_v1_t m_common_header; @@ -329,7 +338,7 @@ struct AMDGpuMetrics_v14_t // Utilization (%) uint16_t m_average_gfx_activity; uint16_t m_average_umc_activity; // memory controller - uint16_t m_vcn_activity[kRSMI_MAX_NUM_VCNS]; // VCN instances activity percent (encode/decode) + uint16_t m_vcn_activity[kRSMI_MAX_NUM_VCNS]; // VCN instances activity percent (encode/decode) // Energy (15.259uJ (2^-16) units) uint64_t m_energy_accumulator; @@ -345,9 +354,9 @@ struct AMDGpuMetrics_v14_t // Link width (number of lanes) and speed (in 0.1 GT/s) uint16_t m_pcie_link_width; - uint16_t m_pcie_link_speed; // in 0.1 GT/s + uint16_t m_pcie_link_speed; // in 0.1 GT/s - // XGMI bus width and bitrate (in Gbps) + // XGMI bus width and bitrate (in Gbps) uint16_t m_xgmi_link_width; uint16_t m_xgmi_link_speed; @@ -358,7 +367,7 @@ struct AMDGpuMetrics_v14_t // PCIE accumulated bandwidth (GB/sec) uint64_t m_pcie_bandwidth_acc; - // PCIE instantaneous bandwidth (GB/sec) + // PCIE instantaneous bandwidth (GB/sec) uint64_t m_pcie_bandwidth_inst; // PCIE L0 to recovery state transition accumulated count @@ -387,8 +396,7 @@ struct AMDGpuMetrics_v14_t uint16_t m_padding; }; -struct AMDGpuMetrics_v15_t -{ +struct AMDGpuMetrics_v15_t { ~AMDGpuMetrics_v15_t() = default; struct AMDGpuMetricsHeader_v1_t m_common_header; @@ -404,7 +412,7 @@ struct AMDGpuMetrics_v15_t // Utilization (%) uint16_t m_average_gfx_activity; uint16_t m_average_umc_activity; // memory controller - uint16_t m_vcn_activity[kRSMI_MAX_NUM_VCNS]; // VCN instances activity percent (encode/decode) + uint16_t m_vcn_activity[kRSMI_MAX_NUM_VCNS]; // VCN instances activity percent (encode/decode) uint16_t m_jpeg_activity[kRSMI_MAX_JPEG_ENGINES]; // JPEG activity percent (encode/decode) // Energy (15.259uJ (2^-16) units) @@ -421,7 +429,7 @@ struct AMDGpuMetrics_v15_t // Link width (number of lanes) and speed (in 0.1 GT/s) uint16_t m_pcie_link_width; - uint16_t m_pcie_link_speed; // in 0.1 GT/s + uint16_t m_pcie_link_speed; // in 0.1 GT/s // XGMI bus width and bitrate (in Gbps) uint16_t m_xgmi_link_width; @@ -468,7 +476,103 @@ struct AMDGpuMetrics_v15_t uint16_t m_padding; }; -using AMGpuMetricsLatest_t = AMDGpuMetrics_v15_t; +struct AMDGpuMetrics_v16_t { + ~AMDGpuMetrics_v16_t() = default; + + struct AMDGpuMetricsHeader_v1_t m_common_header; + + // Temperature (Celsius). It will be zero (0) if unsupported. + uint16_t m_temperature_hotspot; + uint16_t m_temperature_mem; + uint16_t m_temperature_vrsoc; + + // Power (Watts) + uint16_t m_current_socket_power; + + // Utilization (%) + uint16_t m_average_gfx_activity; + uint16_t m_average_umc_activity; // memory controller + + // Energy (15.259uJ (2^-16) units) + uint64_t m_energy_accumulator; + + // Driver attached timestamp (in ns) + uint64_t m_system_clock_counter; + + /* + * Important: bumped up public to uint64_t due to planned size increase + * for newer ASICs + */ + /* Accumulation cycle counter */ + uint32_t m_accumulation_counter; + + /* Accumulated throttler residencies */ + uint32_t m_prochot_residency_acc; + uint32_t m_ppt_residency_acc; + uint32_t m_socket_thm_residency_acc; + uint32_t m_vr_thm_residency_acc; + uint32_t m_hbm_thm_residency_acc; + + // Clock Lock Status. Each bit corresponds to clock instance + uint32_t m_gfxclk_lock_status; + + // Link width (number of lanes) and speed (in 0.1 GT/s) + uint16_t m_pcie_link_width; + uint16_t m_pcie_link_speed; // in 0.1 GT/s + + // XGMI bus width and bitrate (in Gbps) + uint16_t m_xgmi_link_width; + uint16_t m_xgmi_link_speed; + + // Utilization Accumulated (%) + uint32_t m_gfx_activity_acc; + uint32_t m_mem_activity_acc; + + // PCIE accumulated bandwidth (GB/sec) + uint64_t m_pcie_bandwidth_acc; + + // PCIE instantaneous bandwidth (GB/sec) + uint64_t m_pcie_bandwidth_inst; + + // PCIE L0 to recovery state transition accumulated count + uint64_t m_pcie_l0_to_recov_count_acc; + + // PCIE replay accumulated count + uint64_t m_pcie_replay_count_acc; + + // PCIE replay rollover accumulated count + uint64_t m_pcie_replay_rover_count_acc; + + // PCIE NAK sent accumulated count + uint32_t m_pcie_nak_sent_count_acc; + + // PCIE NAK received accumulated count + uint32_t m_pcie_nak_rcvd_count_acc; + + // XGMI accumulated data transfer size(KiloBytes) + uint64_t m_xgmi_read_data_acc[kRSMI_MAX_NUM_XGMI_LINKS]; + uint64_t m_xgmi_write_data_acc[kRSMI_MAX_NUM_XGMI_LINKS]; + + // PMFW attached timestamp (10ns resolution) + uint64_t m_firmware_timestamp; + + // Current clocks (Mhz) + uint16_t m_current_gfxclk[kRSMI_MAX_NUM_GFX_CLKS]; + uint16_t m_current_socclk[kRSMI_MAX_NUM_CLKS]; + uint16_t m_current_vclk0[kRSMI_MAX_NUM_CLKS]; + uint16_t m_current_dclk0[kRSMI_MAX_NUM_CLKS]; + uint16_t m_current_uclk; + + /* Number of current partition */ + uint16_t m_num_partition; + + /* XCP (Graphic Cluster Partitions) metrics stats */ + struct amdgpu_xcp_metrics m_xcp_stats[kRSMI_MAX_NUM_XCP]; + + /* PCIE other end recovery counter */ + uint32_t m_pcie_lc_perf_other_end_recovery; +}; +using AMGpuMetricsLatest_t = AMDGpuMetrics_v16_t; /** * This is GPU Metrics version that gets to public access. @@ -555,8 +659,7 @@ using AMDGpuMetricVersionFlagId_t = uint32_t; * Each Metric Unit (or a set of them) is related to a Metric class. * */ -enum class AMDGpuMetricsClassId_t : AMDGpuMetricTypeId_t -{ +enum class AMDGpuMetricsClassId_t : AMDGpuMetricTypeId_t { kGpuMetricHeader, kGpuMetricTemperature, kGpuMetricUtilization, @@ -569,6 +672,9 @@ enum class AMDGpuMetricsClassId_t : AMDGpuMetricTypeId_t kGpuMetricLinkWidthSpeed, kGpuMetricVoltage, kGpuMetricTimestamp, + kGpuMetricThrottleResidency, + kGpuMetricPartition, + kGpuMetricXcpStats, }; using AMDGpuMetricsClassIdTranslationTbl_t = std::map; @@ -605,8 +711,8 @@ enum class AMDGpuMetricsUnitType_t : AMDGpuMetricTypeId_t kMetricAvgMmActivity, kMetricGfxActivityAccumulator, kMetricMemActivityAccumulator, - kMetricVcnActivity, //v1.4 - kMetricJpegActivity, //v1.5 + kMetricVcnActivity, // v1.4 + kMetricJpegActivity, // v1.5 // kGpuMetricAverageClock counters kMetricAvgGfxClockFrequency, @@ -618,11 +724,11 @@ enum class AMDGpuMetricsUnitType_t : AMDGpuMetricTypeId_t kMetricAvgDClock1Frequency, // kGpuMetricCurrentClock counters - kMetricCurrGfxClock, //v1.4: Changed to multi-valued - kMetricCurrSocClock, //v1.4: Changed to multi-valued + kMetricCurrGfxClock, // v1.4: Changed to multi-valued + kMetricCurrSocClock, // v1.4: Changed to multi-valued kMetricCurrUClock, - kMetricCurrVClock0, //v1.4: Changed to multi-valued - kMetricCurrDClock0, //v1.4: Changed to multi-valued + kMetricCurrVClock0, // v1.4: Changed to multi-valued + kMetricCurrDClock0, // v1.4: Changed to multi-valued kMetricCurrVClock1, kMetricCurrDClock1, @@ -631,7 +737,7 @@ enum class AMDGpuMetricsUnitType_t : AMDGpuMetricTypeId_t kMetricIndepThrottleStatus, // kGpuMetricGfxClkLockStatus counters - kMetricGfxClkLockStatus, //v1.4 + kMetricGfxClkLockStatus, // v1.4 // kGpuMetricCurrentFanSpeed counters kMetricCurrFanSpeed, @@ -639,31 +745,50 @@ enum class AMDGpuMetricsUnitType_t : AMDGpuMetricTypeId_t // kGpuMetricLinkWidthSpeed counters kMetricPcieLinkWidth, kMetricPcieLinkSpeed, - kMetricPcieBandwidthAccumulator, //v1.4 - kMetricPcieBandwidthInst, //v1.4 - kMetricXgmiLinkWidth, //v1.4 - kMetricXgmiLinkSpeed, //v1.4 - kMetricXgmiReadDataAccumulator, //v1.4 - kMetricXgmiWriteDataAccumulator, //v1.4 - kMetricPcieL0RecovCountAccumulator, //v1.4 - kMetricPcieReplayCountAccumulator, //v1.4 - kMetricPcieReplayRollOverCountAccumulator, //v1.4 - kMetricPcieNakSentCountAccumulator, //v1.5 - kMetricPcieNakReceivedCountAccumulator, //v1.5 + kMetricPcieBandwidthAccumulator, // v1.4 + kMetricPcieBandwidthInst, // v1.4 + kMetricXgmiLinkWidth, // v1.4 + kMetricXgmiLinkSpeed, // v1.4 + kMetricXgmiReadDataAccumulator, // v1.4 + kMetricXgmiWriteDataAccumulator, // v1.4 + kMetricPcieL0RecovCountAccumulator, // v1.4 + kMetricPcieReplayCountAccumulator, // v1.4 + kMetricPcieReplayRollOverCountAccumulator, // v1.4 + kMetricPcieNakSentCountAccumulator, // v1.5 + kMetricPcieNakReceivedCountAccumulator, // v1.5 // kGpuMetricPowerEnergy counters kMetricAvgSocketPower, - kMetricCurrSocketPower, //v1.4 - kMetricEnergyAccumulator, //v1.4 + kMetricCurrSocketPower, // v1.4 + kMetricEnergyAccumulator, // v1.4 // kGpuMetricVoltage counters - kMetricVoltageSoc, //v1.3 - kMetricVoltageGfx, //v1.3 - kMetricVoltageMem, //v1.3 + kMetricVoltageSoc, // v1.3 + kMetricVoltageGfx, // v1.3 + kMetricVoltageMem, // v1.3 // kGpuMetricTimestamp counters kMetricTSClockCounter, kMetricTSFirmware, + + // kMetricAccumulationCounter counters + kMetricAccumulationCounter, // v1.6 + kMetricProchotResidencyAccumulator, // v1.6 + kMetricPPTResidencyAccumulator, // v1.6 + kMetricSocketThmResidencyAccumulator, // v1.6 + kMetricVRThmResidencyAccumulator, // v1.6 + kMetricHBMThmResidencyAccumulator, // v1.6 + + // kGpuMetricPartition + kGpuMetricNumPartition, // v1.6 + + // kGpuMetricXcpStats + kMetricGfxBusyInst, // v1.6 + kMetricJpegBusy, // v1.6 + kMetricVcnBusy, // v1.6 + kMetricGfxBusyAcc, // v1.6 + + kMetricPcieLCPerfOtherEndRecov, // v1.6 }; using AMDGpuMetricsUnitTypeTranslationTbl_t = std::map; @@ -676,14 +801,14 @@ enum class AMDGpuMetricsDataType_t : AMDGpuMetricsDataTypeId_t kUInt64, }; -struct AMDGpuDynamicMetricsValue_t -{ +struct AMDGpuDynamicMetricsValue_t { uint64_t m_value; std::string m_info; AMDGpuMetricsDataType_t m_original_type; }; using AMDGpuDynamicMetricTblValues_t = std::vector; -using AMDGpuDynamicMetricsTbl_t = std::map>; +using AMDGpuDynamicMetricsTbl_t = std::map>; /* @@ -700,13 +825,13 @@ enum class AMDGpuMetricVersionFlags_t : AMDGpuMetricVersionFlagId_t kGpuMetricV13 = (0x1 << 3), kGpuMetricV14 = (0x1 << 4), kGpuMetricV15 = (0x1 << 5), + kGpuMetricV16 = (0x1 << 6), }; using AMDGpuMetricVersionTranslationTbl_t = std::map; using GpuMetricTypePtr_t = std::shared_ptr; -class GpuMetricsBase_t -{ - public: +class GpuMetricsBase_t { + public: virtual ~GpuMetricsBase_t() = default; virtual size_t sizeof_metric_table() = 0; virtual GpuMetricTypePtr_t get_metrics_table() = 0; @@ -714,30 +839,32 @@ class GpuMetricsBase_t virtual AMDGpuMetricVersionFlags_t get_gpu_metrics_version_used() = 0; virtual rsmi_status_t populate_metrics_dynamic_tbl() = 0; virtual AMGpuMetricsPublicLatestTupl_t copy_internal_to_external_metrics() = 0; + virtual void set_device_id(uint32_t device_id) { m_device_id = device_id; } + virtual void set_partition_id(uint32_t partition_id) { m_partition_id = partition_id; } virtual AMDGpuDynamicMetricsTbl_t get_metrics_dynamic_tbl() { return m_metrics_dynamic_tbl; } - protected: + protected: AMDGpuDynamicMetricsTbl_t m_metrics_dynamic_tbl; uint64_t m_metrics_timestamp; + uint32_t m_device_id; + uint32_t m_partition_id; }; using GpuMetricsBasePtr = std::shared_ptr; using AMDGpuMetricFactories_t = const std::map; -class GpuMetricsBase_v11_t final : public GpuMetricsBase_t -{ - public: +class GpuMetricsBase_v11_t final : public GpuMetricsBase_t { + public: virtual ~GpuMetricsBase_v11_t() = default; size_t sizeof_metric_table() override { return sizeof(AMDGpuMetrics_v11_t); } - GpuMetricTypePtr_t get_metrics_table() override - { + GpuMetricTypePtr_t get_metrics_table() override { if (!m_gpu_metric_ptr) { m_gpu_metric_ptr.reset(&m_gpu_metrics_tbl, [](AMDGpuMetrics_v11_t*){}); } @@ -745,13 +872,11 @@ class GpuMetricsBase_v11_t final : public GpuMetricsBase_t return m_gpu_metric_ptr; } - void dump_internal_metrics_table() override - { + void dump_internal_metrics_table() override { return; } - AMDGpuMetricVersionFlags_t get_gpu_metrics_version_used() override - { + AMDGpuMetricVersionFlags_t get_gpu_metrics_version_used() override { return AMDGpuMetricVersionFlags_t::kGpuMetricV11; } @@ -759,23 +884,20 @@ class GpuMetricsBase_v11_t final : public GpuMetricsBase_t AMGpuMetricsPublicLatestTupl_t copy_internal_to_external_metrics() override; - private: + private: AMDGpuMetrics_v11_t m_gpu_metrics_tbl; std::shared_ptr m_gpu_metric_ptr; - }; -class GpuMetricsBase_v12_t final : public GpuMetricsBase_t -{ - public: +class GpuMetricsBase_v12_t final : public GpuMetricsBase_t { + public: ~GpuMetricsBase_v12_t() = default; size_t sizeof_metric_table() override { return sizeof(AMDGpuMetrics_v12_t); } - GpuMetricTypePtr_t get_metrics_table() override - { + GpuMetricTypePtr_t get_metrics_table() override { if (!m_gpu_metric_ptr) { m_gpu_metric_ptr.reset(&m_gpu_metrics_tbl, [](AMDGpuMetrics_v12_t*){}); } @@ -783,36 +905,31 @@ class GpuMetricsBase_v12_t final : public GpuMetricsBase_t return m_gpu_metric_ptr; } - void dump_internal_metrics_table() override - { + void dump_internal_metrics_table() override { return; } - AMDGpuMetricVersionFlags_t get_gpu_metrics_version_used() override - { + AMDGpuMetricVersionFlags_t get_gpu_metrics_version_used() override { return AMDGpuMetricVersionFlags_t::kGpuMetricV12; } rsmi_status_t populate_metrics_dynamic_tbl() override; AMGpuMetricsPublicLatestTupl_t copy_internal_to_external_metrics() override; - private: + private: AMDGpuMetrics_v12_t m_gpu_metrics_tbl; std::shared_ptr m_gpu_metric_ptr; - }; -class GpuMetricsBase_v13_t final : public GpuMetricsBase_t -{ - public: +class GpuMetricsBase_v13_t final : public GpuMetricsBase_t { + public: ~GpuMetricsBase_v13_t() = default; size_t sizeof_metric_table() override { return sizeof(AMDGpuMetrics_v13_t); } - GpuMetricTypePtr_t get_metrics_table() override - { + GpuMetricTypePtr_t get_metrics_table() override { if (!m_gpu_metric_ptr) { m_gpu_metric_ptr.reset(&m_gpu_metrics_tbl, [](AMDGpuMetrics_v13_t*){}); } @@ -822,8 +939,7 @@ class GpuMetricsBase_v13_t final : public GpuMetricsBase_t void dump_internal_metrics_table() override; - AMDGpuMetricVersionFlags_t get_gpu_metrics_version_used() override - { + AMDGpuMetricVersionFlags_t get_gpu_metrics_version_used() override { return AMDGpuMetricVersionFlags_t::kGpuMetricV13; } @@ -831,23 +947,20 @@ class GpuMetricsBase_v13_t final : public GpuMetricsBase_t AMGpuMetricsPublicLatestTupl_t copy_internal_to_external_metrics() override; - private: + private: AMDGpuMetrics_v13_t m_gpu_metrics_tbl; std::shared_ptr m_gpu_metric_ptr; - }; -class GpuMetricsBase_v14_t final : public GpuMetricsBase_t -{ - public: +class GpuMetricsBase_v14_t final : public GpuMetricsBase_t { + public: ~GpuMetricsBase_v14_t() = default; size_t sizeof_metric_table() override { return sizeof(AMDGpuMetrics_v14_t); } - GpuMetricTypePtr_t get_metrics_table() override - { + GpuMetricTypePtr_t get_metrics_table() override { if (!m_gpu_metric_ptr) { m_gpu_metric_ptr.reset(&m_gpu_metrics_tbl, [](AMDGpuMetrics_v14_t*){}); } @@ -857,8 +970,7 @@ class GpuMetricsBase_v14_t final : public GpuMetricsBase_t void dump_internal_metrics_table() override; - AMDGpuMetricVersionFlags_t get_gpu_metrics_version_used() override - { + AMDGpuMetricVersionFlags_t get_gpu_metrics_version_used() override { return AMDGpuMetricVersionFlags_t::kGpuMetricV14; } @@ -866,23 +978,20 @@ class GpuMetricsBase_v14_t final : public GpuMetricsBase_t AMGpuMetricsPublicLatestTupl_t copy_internal_to_external_metrics() override; - private: + private: AMDGpuMetrics_v14_t m_gpu_metrics_tbl; std::shared_ptr m_gpu_metric_ptr; - }; -class GpuMetricsBase_v15_t final : public GpuMetricsBase_t -{ - public: +class GpuMetricsBase_v15_t final : public GpuMetricsBase_t { + public: ~GpuMetricsBase_v15_t() = default; size_t sizeof_metric_table() override { return sizeof(AMDGpuMetrics_v15_t); } - GpuMetricTypePtr_t get_metrics_table() override - { + GpuMetricTypePtr_t get_metrics_table() override { if (!m_gpu_metric_ptr) { m_gpu_metric_ptr.reset(&m_gpu_metrics_tbl, [](AMDGpuMetrics_v15_t*){}); } @@ -892,8 +1001,7 @@ class GpuMetricsBase_v15_t final : public GpuMetricsBase_t void dump_internal_metrics_table() override; - AMDGpuMetricVersionFlags_t get_gpu_metrics_version_used() override - { + AMDGpuMetricVersionFlags_t get_gpu_metrics_version_used() override { return AMDGpuMetricVersionFlags_t::kGpuMetricV15; } @@ -901,20 +1009,51 @@ class GpuMetricsBase_v15_t final : public GpuMetricsBase_t AMGpuMetricsPublicLatestTupl_t copy_internal_to_external_metrics() override; - private: + private: AMDGpuMetrics_v15_t m_gpu_metrics_tbl; std::shared_ptr m_gpu_metric_ptr; +}; +class GpuMetricsBase_v16_t final : public GpuMetricsBase_t { + public: + ~GpuMetricsBase_v16_t() = default; + + size_t sizeof_metric_table() override { + return sizeof(AMDGpuMetrics_v16_t); + } + + GpuMetricTypePtr_t get_metrics_table() override { + if (!m_gpu_metric_ptr) { + m_gpu_metric_ptr.reset(&m_gpu_metrics_tbl, [](AMDGpuMetrics_v16_t*){}); + } + assert(m_gpu_metric_ptr != nullptr); + return m_gpu_metric_ptr; + } + + void dump_internal_metrics_table() override; + + AMDGpuMetricVersionFlags_t get_gpu_metrics_version_used() override { + return AMDGpuMetricVersionFlags_t::kGpuMetricV16; + } + + rsmi_status_t populate_metrics_dynamic_tbl() override; + AMGpuMetricsPublicLatestTupl_t copy_internal_to_external_metrics() override; + + private: + AMDGpuMetrics_v16_t m_gpu_metrics_tbl; + std::shared_ptr m_gpu_metric_ptr; }; template -rsmi_status_t rsmi_dev_gpu_metrics_info_query(uint32_t dv_ind, AMDGpuMetricsUnitType_t metric_counter, T& metric_value); +rsmi_status_t rsmi_dev_gpu_metrics_info_query(uint32_t dv_ind, + AMDGpuMetricsUnitType_t metric_counter, T& metric_value); -} // namespace amd::smi +} // namespace amd::smi rsmi_status_t -rsmi_dev_gpu_metrics_header_info_get(uint32_t dv_ind, metrics_table_header_t& header_value); +rsmi_dev_gpu_metrics_header_info_get(uint32_t dv_ind, + metrics_table_header_t& header_value); -#endif // ROCM_SMI_ROCM_SMI_GPU_METRICS_H_ +#endif // ROCM_SMI_ROCM_SMI_GPU_METRICS_H_ diff --git a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_utils.h b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_utils.h index 431c60698c..abf5e98831 100755 --- a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_utils.h +++ b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_utils.h @@ -48,14 +48,18 @@ #include #include #include +#include #include +#include #include +#include #include #include #include #include #include #include +#include #include "rocm_smi/rocm_smi_device.h" @@ -599,6 +603,74 @@ using TextFileTagContents_t = TagTextContents_t; +// +// Note: Output iterator that inserts a delimiter between elements. +// +template> +class ostream_joiner { + public: + using Char_t = CharType; + using Traits_t = TraitsType; + using Ostream_t = std::basic_ostream; + using iterator_category = std::output_iterator_tag; + using value_type = void; + using difference_type = void; + using pointer = void; + using reference = void; + + + ostream_joiner(Ostream_t* outstream, + const DelimiterType& delimiter) noexcept + (std::is_nothrow_copy_constructible_v) + : m_outstream(outstream), m_delimiter(delimiter) {} + + ostream_joiner(Ostream_t* outstream, DelimiterType&& delimiter) noexcept + (std::is_nothrow_move_constructible_v) + : m_outstream(outstream), m_delimiter(std::move(delimiter)) {} + + template ostream_joiner& operator=(const ValueType& value) { + if (!m_is_first) { + *m_outstream << m_delimiter; + } + this->m_is_first = false; + this->m_value_count++; + + if ((m_value_count % kMAX_VALUES_PER_LINE) == 0) { + *m_outstream << "\n" << value; + this->m_value_count = 0; + } else { + *m_outstream << value; + } + + return *this; + } + + ostream_joiner& operator*() noexcept { return *this; } + ostream_joiner& operator++() noexcept { return *this; } + ostream_joiner& operator++(int) noexcept { return *this; } + + + private: + Ostream_t* m_outstream; + DelimiterType m_delimiter; + bool m_is_first = true; + uint32_t m_value_count = 0; + const uint32_t kMAX_VALUES_PER_LINE = 9; +}; + +/// Object generator for ostream_joiner. +template +inline ostream_joiner, CharType, TraitsType> + make_ostream_joiner(std::basic_ostream* outstream, + DelimiterType&& delimiter) { + return { + outstream, + std::forward(delimiter) + }; +} + + } // namespace smi } // namespace amd diff --git a/projects/rocm-smi-lib/python_smi_tools/rocm_smi.py b/projects/rocm-smi-lib/python_smi_tools/rocm_smi.py index fa7d2a2323..93435fbaaa 100755 --- a/projects/rocm-smi-lib/python_smi_tools/rocm_smi.py +++ b/projects/rocm-smi-lib/python_smi_tools/rocm_smi.py @@ -24,6 +24,7 @@ import trace from io import StringIO from time import ctime from subprocess import check_output +from enum import IntEnum from typing import TYPE_CHECKING # only used for type checking @@ -48,9 +49,9 @@ except ImportError: # Minor version - Increment when adding a new feature, set to 0 when major is incremented # Patch version - Increment when adding a fix, set to 0 when minor is incremented # Hash version - Shortened commit hash. Print here and not with lib for consistency with amd-smi -SMI_MAJ = 2 -SMI_MIN = 3 -SMI_PAT = 1 +SMI_MAJ = 3 +SMI_MIN = 0 +SMI_PAT = 0 # SMI_HASH is provided by rsmiBindings __version__ = '%s.%s.%s+%s' % (SMI_MAJ, SMI_MIN, SMI_PAT, SMI_HASH) @@ -856,7 +857,7 @@ def printEventList(device, delay, eventList): print2DArray([['\rGPU[%d]:\t' % (data.dv_ind), ctime().split()[3], notification_type_names[data.event.value - 1], data.message.decode('utf8') + '\r']]) -def printLog(device, metricName, value=None, extraSpace=False, useItalics=False): +def printLog(device, metricName, value=None, extraSpace=False, useItalics=False, xcp=None): """ Print out to the SMI log :param device: DRM device identifier @@ -878,7 +879,10 @@ def printLog(device, metricName, value=None, extraSpace=False, useItalics=False) formatJson(device, str(metricName)) return if value is not None: - logstr = 'GPU[%s]\t\t: %s: %s' % (device, metricName, value) + if xcp == None: + logstr = 'GPU[%s]\t\t: %s: %s' % (device, metricName, value) + else: + logstr = 'GPU[%s] XCP[%s]\t: %s: %s' % (device, xcp, metricName, value) else: logstr = 'GPU[%s]\t\t: %s' % (device, metricName) if device is None: @@ -3544,6 +3548,318 @@ def showMemoryPartition(deviceList): printErrLog(device, 'Failed to retrieve current memory partition, even though device supports it.') printLogSpacer() +class UIntegerTypes(IntEnum): + UINT8_T = 0xFF + UINT16_T = 0xFFFF + UINT32_T = 0xFFFFFFFF + UINT64_T = 0xFFFFFFFFFFFFFFFF + +def validateIfMaxUint(valToCheck, uintType: UIntegerTypes): + return_val = "N/A" + if not isinstance(valToCheck, list): + if valToCheck == uintType: + return return_val + else: + return valToCheck + else: + return_val = valToCheck + for idx, v in enumerate(valToCheck): + if v == uintType: + return_val[idx] = "N/A" + return return_val + +def showGPUMetrics(deviceList): + """ Returns the gpu metrics for a list of devices + + :param deviceList: List of DRM devices (can be a single-item list) + """ + printLogSpacer(' GPU Metrics ') + gpu_metrics = rsmi_gpu_metrics_t() + temp_unit="C" + power_unit="W" + energy_unit="15.259uJ (2^-16)" + volt_unit="mV" + clock_unit="MHz" + fan_speed="rpm" + percent_unit="%" + pcie_acc_unit="GB/s" + pcie_lanes_unit="Lanes" + pcie_speed_unit="0.1 GT/s" + xgmi_speed="Gbps" + xgmi_data_sz="kB" + time_unit="ns" + time_unit_10="10ns resolution" + count="Count" + no_unit = None + + for device in deviceList: + ret = rocmsmi.rsmi_dev_gpu_metrics_info_get(device, byref(gpu_metrics)) + metrics = { + "common_header": "N/A" + } + if rsmi_ret_ok(ret, device, 'rsmi_dev_gpu_metrics_info_get',silent=True): + metrics = { + "common_header": { + "version": float(str(gpu_metrics.common_header.format_revision) + "." + + str(gpu_metrics.common_header.content_revision)), + "size": gpu_metrics.common_header.structure_size + }, "temperature_edge": { + "value": validateIfMaxUint(gpu_metrics.temperature_edge, UIntegerTypes.UINT16_T), + "unit": temp_unit, + }, "temperature_hotspot": { + "value": validateIfMaxUint(gpu_metrics.temperature_hotspot, UIntegerTypes.UINT16_T), + "unit": temp_unit, + }, "temperature_mem": { + "value": validateIfMaxUint(gpu_metrics.temperature_mem, UIntegerTypes.UINT16_T), + "unit": temp_unit, + }, "temperature_vrgfx": { + "value": validateIfMaxUint(gpu_metrics.temperature_vrgfx, UIntegerTypes.UINT16_T), + "unit": temp_unit, + }, "temperature_vrsoc": { + "value": validateIfMaxUint(gpu_metrics.temperature_vrsoc, UIntegerTypes.UINT16_T), + "unit": temp_unit, + }, "temperature_vrmem": { + "value": validateIfMaxUint(gpu_metrics.temperature_vrmem, UIntegerTypes.UINT16_T), + "unit": temp_unit, + }, "average_gfx_activity": { + "value": validateIfMaxUint(gpu_metrics.average_gfx_activity, UIntegerTypes.UINT16_T), + "unit": percent_unit, + }, "average_umc_activity": { + "value": validateIfMaxUint(gpu_metrics.average_umc_activity, UIntegerTypes.UINT16_T), + "unit": percent_unit, + }, "average_mm_activity": { + "value": validateIfMaxUint(gpu_metrics.average_mm_activity, UIntegerTypes.UINT16_T), + "unit": percent_unit, + }, "average_socket_power": { + "value": validateIfMaxUint(gpu_metrics.average_socket_power, UIntegerTypes.UINT16_T), + "unit": power_unit, + }, "energy_accumulator": { + "value": validateIfMaxUint(gpu_metrics.energy_accumulator, UIntegerTypes.UINT64_T), + "unit": energy_unit, + }, "system_clock_counter": { + "value": validateIfMaxUint(gpu_metrics.system_clock_counter, UIntegerTypes.UINT64_T), + "unit": time_unit, + }, "average_gfxclk_frequency": { + "value": validateIfMaxUint(gpu_metrics.average_gfxclk_frequency, UIntegerTypes.UINT16_T), + "unit": clock_unit, + }, "average_socclk_frequency": { + "value": validateIfMaxUint(gpu_metrics.average_socclk_frequency, UIntegerTypes.UINT16_T), + "unit": clock_unit, + }, "average_uclk_frequency": { + "value": validateIfMaxUint(gpu_metrics.average_uclk_frequency, UIntegerTypes.UINT16_T), + "unit": clock_unit, + }, "average_vclk0_frequency": { + "value": validateIfMaxUint(gpu_metrics.average_vclk0_frequency, UIntegerTypes.UINT16_T), + "unit": clock_unit, + }, "average_dclk0_frequency": { + "value": validateIfMaxUint(gpu_metrics.average_dclk0_frequency, UIntegerTypes.UINT16_T), + "unit": clock_unit, + }, "average_vclk1_frequency": { + "value": validateIfMaxUint(gpu_metrics.average_vclk1_frequency, UIntegerTypes.UINT16_T), + "unit": clock_unit, + }, "average_dclk1_frequency": { + "value": validateIfMaxUint(gpu_metrics.average_dclk1_frequency, UIntegerTypes.UINT16_T), + "unit": clock_unit, + }, "current_gfxclk": { + "value": validateIfMaxUint(gpu_metrics.current_gfxclk, UIntegerTypes.UINT16_T), + "unit": clock_unit, + }, "current_socclk": { + "value": validateIfMaxUint(gpu_metrics.current_socclk, UIntegerTypes.UINT16_T), + "unit": clock_unit, + }, "current_uclk": { + "value": validateIfMaxUint(gpu_metrics.current_uclk, UIntegerTypes.UINT16_T), + "unit": clock_unit, + }, "current_vclk0": { + "value": validateIfMaxUint(gpu_metrics.current_vclk0, UIntegerTypes.UINT16_T), + "unit": clock_unit, + }, "current_dclk0": { + "value": validateIfMaxUint(gpu_metrics.current_dclk0, UIntegerTypes.UINT16_T), + "unit": clock_unit, + }, "current_vclk1": { + "value": validateIfMaxUint(gpu_metrics.current_vclk1, UIntegerTypes.UINT16_T), + "unit": clock_unit, + }, "current_dclk1": { + "value": validateIfMaxUint(gpu_metrics.current_dclk1, UIntegerTypes.UINT16_T), + "unit": clock_unit, + }, "throttle_status": { + "value": validateIfMaxUint(gpu_metrics.throttle_status, UIntegerTypes.UINT32_T), + "unit": no_unit, + }, "current_fan_speed": { + "value": validateIfMaxUint(gpu_metrics.current_fan_speed, UIntegerTypes.UINT16_T), + "unit": fan_speed, + }, "pcie_link_width": { + "value": validateIfMaxUint(gpu_metrics.pcie_link_width, UIntegerTypes.UINT16_T), + "unit": pcie_lanes_unit, + }, "pcie_link_speed": { + "value": validateIfMaxUint(gpu_metrics.pcie_link_speed, UIntegerTypes.UINT16_T), + "unit": pcie_speed_unit, + }, "gfx_activity_acc": { + "value": validateIfMaxUint(gpu_metrics.gfx_activity_acc, UIntegerTypes.UINT32_T), + "unit": percent_unit, + }, "mem_activity_acc": { + "value": validateIfMaxUint(gpu_metrics.mem_activity_acc, UIntegerTypes.UINT32_T), + "unit": percent_unit, + }, "temperature_hbm": { + "value": validateIfMaxUint(list(gpu_metrics.temperature_hbm), UIntegerTypes.UINT16_T), + "unit": temp_unit, + }, "firmware_timestamp": { + "value": validateIfMaxUint(gpu_metrics.firmware_timestamp, UIntegerTypes.UINT64_T), + "unit": time_unit_10, + }, "voltage_soc": { + "value": validateIfMaxUint(gpu_metrics.voltage_soc, UIntegerTypes.UINT16_T), + "unit": volt_unit, + }, "voltage_gfx": { + "value": validateIfMaxUint(gpu_metrics.voltage_gfx, UIntegerTypes.UINT16_T), + "unit": volt_unit, + }, "voltage_mem": { + "value": validateIfMaxUint(gpu_metrics.voltage_mem, UIntegerTypes.UINT16_T), + "unit": volt_unit, + }, "indep_throttle_status": { + "value": validateIfMaxUint(gpu_metrics.indep_throttle_status, UIntegerTypes.UINT64_T), + "unit": no_unit, + }, "current_socket_power": { + "value": validateIfMaxUint(gpu_metrics.current_socket_power, UIntegerTypes.UINT16_T), + "unit": power_unit, + }, "vcn_activity": { + "value": validateIfMaxUint(list(gpu_metrics.vcn_activity), UIntegerTypes.UINT16_T), + "unit": percent_unit, + }, "gfxclk_lock_status": { + "value": validateIfMaxUint(gpu_metrics.gfxclk_lock_status, UIntegerTypes.UINT32_T), + "unit": no_unit, + }, "xgmi_link_width": { + "value": validateIfMaxUint(gpu_metrics.xgmi_link_width, UIntegerTypes.UINT16_T), + "unit": no_unit, + }, "xgmi_link_speed": { + "value": validateIfMaxUint(gpu_metrics.xgmi_link_speed, UIntegerTypes.UINT16_T), + "unit": xgmi_speed, + }, "pcie_bandwidth_acc": { + "value": validateIfMaxUint(gpu_metrics.pcie_bandwidth_acc, UIntegerTypes.UINT64_T), + "unit": pcie_acc_unit, + }, "pcie_bandwidth_inst": { + "value": validateIfMaxUint(gpu_metrics.pcie_bandwidth_inst, UIntegerTypes.UINT64_T), + "unit": pcie_acc_unit, + }, "pcie_l0_to_recov_count_acc": { + "value": validateIfMaxUint(gpu_metrics.pcie_l0_to_recov_count_acc, UIntegerTypes.UINT64_T), + "unit": count, + }, "pcie_replay_count_acc": { + "value": validateIfMaxUint(gpu_metrics.pcie_replay_count_acc, UIntegerTypes.UINT64_T), + "unit": count, + }, "pcie_replay_rover_count_acc": { + "value": validateIfMaxUint(gpu_metrics.pcie_replay_rover_count_acc, UIntegerTypes.UINT64_T), + "unit": count, + }, "xgmi_read_data_acc": { + "value": validateIfMaxUint(list(gpu_metrics.xgmi_read_data_acc), UIntegerTypes.UINT64_T), + "unit": xgmi_data_sz, + }, "xgmi_write_data_acc": { + "value": validateIfMaxUint(list(gpu_metrics.xgmi_write_data_acc), UIntegerTypes.UINT64_T), + "unit": xgmi_data_sz, + }, "current_gfxclks": { + "value": validateIfMaxUint(list(gpu_metrics.current_gfxclks), UIntegerTypes.UINT16_T), + "unit": clock_unit, + }, "current_socclks": { + "value": validateIfMaxUint(list(gpu_metrics.current_socclks), UIntegerTypes.UINT16_T), + "unit": clock_unit, + }, "current_vclk0s": { + "value": validateIfMaxUint(list(gpu_metrics.current_vclk0s), UIntegerTypes.UINT16_T), + "unit": clock_unit, + }, "current_dclk0s": { + "value": validateIfMaxUint(list(gpu_metrics.current_dclk0s), UIntegerTypes.UINT16_T), + "unit": clock_unit, + }, "jpeg_activity": { + "value": validateIfMaxUint(list(gpu_metrics.jpeg_activity), UIntegerTypes.UINT16_T), + "unit": percent_unit, + }, "pcie_nak_sent_count_acc": { + "value": validateIfMaxUint(gpu_metrics.pcie_nak_sent_count_acc, UIntegerTypes.UINT32_T), + "unit": count, + }, "pcie_nak_rcvd_count_acc": { + "value": validateIfMaxUint(gpu_metrics.pcie_nak_rcvd_count_acc, UIntegerTypes.UINT32_T), + "unit": count, + }, "accumulation_counter": { + "value": validateIfMaxUint(gpu_metrics.accumulation_counter, UIntegerTypes.UINT64_T), + "unit": count, + }, "prochot_residency_acc": { + "value": validateIfMaxUint(gpu_metrics.prochot_residency_acc, UIntegerTypes.UINT64_T), + "unit": count, + }, "ppt_residency_acc": { + "value": validateIfMaxUint(gpu_metrics.ppt_residency_acc, UIntegerTypes.UINT64_T), + "unit": count, + }, "socket_thm_residency_acc": { + "value": validateIfMaxUint(gpu_metrics.socket_thm_residency_acc, UIntegerTypes.UINT64_T), + "unit": count, + }, "vr_thm_residency_acc": { + "value": validateIfMaxUint(gpu_metrics.vr_thm_residency_acc, UIntegerTypes.UINT64_T), + "unit": count, + }, "hbm_thm_residency_acc": { + "value": validateIfMaxUint(gpu_metrics.hbm_thm_residency_acc, UIntegerTypes.UINT64_T), + "unit": count, + }, + "pcie_lc_perf_other_end_recovery": { + "value": validateIfMaxUint(gpu_metrics.pcie_lc_perf_other_end_recovery, UIntegerTypes.UINT32_T), + "unit": count, + }, + "num_partition": { + "value": validateIfMaxUint(gpu_metrics.num_partition, UIntegerTypes.UINT16_T), + "unit": no_unit, + }, + "xcp_stats.gfx_busy_inst": { + "value": gpu_metrics.xcp_stats, + "unit": percent_unit, + }, + "xcp_stats.jpeg_busy": { + "value": gpu_metrics.xcp_stats, + "unit": percent_unit, + }, + "xcp_stats.vcn_busy": { + "value": gpu_metrics.xcp_stats, + "unit": percent_unit, + }, + "xcp_stats.gfx_busy_acc": { + "value": gpu_metrics.xcp_stats, + "unit": percent_unit, + }, + } + + printLog(device, 'Metric Version and Size (Bytes)', + str(metrics["common_header"]["version"]) + " " + str(metrics["common_header"]["size"])) + for k,v in metrics.items(): + if k != "common_header" and 'xcp_stats' not in k: + if v["unit"] != None: + printLog(device, k + " (" + str(v["unit"]) + ")", str(v["value"])) + elif v["unit"] == None: + printLog(device, k, str(v["value"])) + if 'xcp_stats.gfx_busy_inst' in k: + for curr_xcp, item in enumerate(v['value']): + print_xcp_detail = [] + for _, val in enumerate(item.gfx_busy_inst): + print_xcp_detail.append(validateIfMaxUint(val, UIntegerTypes.UINT32_T)) + printLog(device, k + " (" + str(v["unit"]) + ")", str(print_xcp_detail), xcp=str(curr_xcp)) + if 'xcp_stats.jpeg_busy' in k: + for curr_xcp, item in enumerate(v['value']): + print_xcp_detail = [] + for _, val in enumerate(item.jpeg_busy): + print_xcp_detail.append(validateIfMaxUint(val, UIntegerTypes.UINT16_T)) + printLog(device, k + " (" + str(v["unit"]) + ")", str(print_xcp_detail), xcp=str(curr_xcp)) + if 'xcp_stats.vcn_busy' in k: + for curr_xcp, item in enumerate(v['value']): + print_xcp_detail = [] + for _, val in enumerate(item.vcn_busy): + print_xcp_detail.append(validateIfMaxUint(val, UIntegerTypes.UINT16_T)) + printLog(device, k + " (" + str(v["unit"]) + ")", str(print_xcp_detail), xcp=str(curr_xcp)) + if 'xcp_stats.gfx_busy_acc' in k: + for curr_xcp, item in enumerate(v['value']): + print_xcp_detail = [] + for _, val in enumerate(item.gfx_busy_acc): + print_xcp_detail.append(validateIfMaxUint(val, UIntegerTypes.UINT64_T)) + printLog(device, k + " (" + str(v["unit"]) + ")", str(print_xcp_detail), xcp=str(curr_xcp)) + + if int(device) < (len(deviceList) - 1): + printLogSpacer() + elif ret == rsmi_status_t.RSMI_STATUS_NOT_SUPPORTED: + printLog(device, 'Not supported on the given system', None) + else: + rsmi_ret_ok(ret, device, 'get_gpu_metrics') + printErrLog(device, 'Failed to retrieve GPU metrics, metric version may not be supported for this device.') + printLogSpacer() def checkAmdGpus(deviceList): """ Check if there are any AMD GPUs being queried, @@ -3913,6 +4229,7 @@ if __name__ == '__main__': groupDisplay.add_argument('--shownodesbw', help='Shows the numa nodes ', action='store_true') groupDisplay.add_argument('--showcomputepartition', help='Shows current compute partitioning ', action='store_true') groupDisplay.add_argument('--showmemorypartition', help='Shows current memory partition ', action='store_true') + groupDisplay.add_argument('--showmetrics', help='Show current gpu metric data ', action='store_true') groupActionReset.add_argument('-r', '--resetclocks', help='Reset clocks and OverDrive to default', action='store_true') @@ -4079,6 +4396,7 @@ if __name__ == '__main__': args.showvc = True args.showcomputepartition = True args.showmemorypartition = True + args.showmetrics = True if not PRINT_JSON: args.showprofile = True @@ -4209,6 +4527,8 @@ if __name__ == '__main__': showComputePartition(deviceList) if args.showmemorypartition: showMemoryPartition(deviceList) + if args.showmetrics: + showGPUMetrics(deviceList) if args.setclock: setClocks(deviceList, args.setclock[0], [int(args.setclock[1])]) if args.setsclk: diff --git a/projects/rocm-smi-lib/python_smi_tools/rsmiBindings.py b/projects/rocm-smi-lib/python_smi_tools/rsmiBindings.py index 45011fa42b..6a9aa4219a 100644 --- a/projects/rocm-smi-lib/python_smi_tools/rsmiBindings.py +++ b/projects/rocm-smi-lib/python_smi_tools/rsmiBindings.py @@ -642,3 +642,102 @@ rsmi_power_type_dict = { 1: 'CURRENT SOCKET', 0xFFFFFFFF: 'INVALID_POWER_TYPE' } + +class metrics_table_header_t(Structure): + pass + +# metrics_table_header_t._pack_ = 1 # source:False +metrics_table_header_t._fields_ = [ + ('structure_size', c_uint16), + ('format_revision', c_uint8), + ('content_revision', c_uint8), +] +amd_metrics_table_header_t = metrics_table_header_t + +class amdgpu_xcp_metrics_t(Structure): + pass + +# amdgpu_xcp_metrics_t._pack_ = 1 # source:False +amdgpu_xcp_metrics_t._fields_ = [ + ('gfx_busy_inst', c_uint32 * 8), + ('jpeg_busy', c_uint16 * 32), + ('vcn_busy', c_uint16 * 4), + ('gfx_busy_acc', c_uint64 * 8), +] +xcp_stats_t = amdgpu_xcp_metrics_t + +class rsmi_gpu_metrics_t(Structure): + pass + + +# rsmi_gpu_metrics_t._pack_ = 1 # source:False +rsmi_gpu_metrics_t._fields_ = [ + ('common_header', amd_metrics_table_header_t), + ('temperature_edge', c_uint16), + ('temperature_hotspot', c_uint16), + ('temperature_mem', c_uint16), + ('temperature_vrgfx', c_uint16), + ('temperature_vrsoc', c_uint16), + ('temperature_vrmem', c_uint16), + ('average_gfx_activity', c_uint16), + ('average_umc_activity', c_uint16), + ('average_mm_activity', c_uint16), + ('average_socket_power', c_uint16), + ('energy_accumulator', c_uint64), + ('system_clock_counter', c_uint64), + ('average_gfxclk_frequency', c_uint16), + ('average_socclk_frequency', c_uint16), + ('average_uclk_frequency', c_uint16), + ('average_vclk0_frequency', c_uint16), + ('average_dclk0_frequency', c_uint16), + ('average_vclk1_frequency', c_uint16), + ('average_dclk1_frequency', c_uint16), + ('current_gfxclk', c_uint16), + ('current_socclk', c_uint16), + ('current_uclk', c_uint16), + ('current_vclk0', c_uint16), + ('current_dclk0', c_uint16), + ('current_vclk1', c_uint16), + ('current_dclk1', c_uint16), + ('throttle_status', c_uint32), + ('current_fan_speed', c_uint16), + ('pcie_link_width', c_uint16), + ('pcie_link_speed', c_uint16), + ('gfx_activity_acc', c_uint32), + ('mem_activity_acc', c_uint32), + ('temperature_hbm', c_uint16 * 4), + ('firmware_timestamp', c_uint64), + ('voltage_soc', c_uint16), + ('voltage_gfx', c_uint16), + ('voltage_mem', c_uint16), + ('indep_throttle_status', c_uint64), + ('current_socket_power', c_uint16), + ('vcn_activity', c_uint16 * 4), + ('gfxclk_lock_status', c_uint32), + ('xgmi_link_width', c_uint16), + ('xgmi_link_speed', c_uint16), + ('pcie_bandwidth_acc', c_uint64), + ('pcie_bandwidth_inst', c_uint64), + ('pcie_l0_to_recov_count_acc', c_uint64), + ('pcie_replay_count_acc', c_uint64), + ('pcie_replay_rover_count_acc', c_uint64), + ('xgmi_read_data_acc', c_uint64 * 8), + ('xgmi_write_data_acc', c_uint64 * 8), + ('current_gfxclks', c_uint16 * 8), + ('current_socclks', c_uint16 * 4), + ('current_vclk0s', c_uint16 * 4), + ('current_dclk0s', c_uint16 * 4), + ('jpeg_activity', c_uint16 * 32), + ('pcie_nak_sent_count_acc', c_uint32), + ('pcie_nak_rcvd_count_acc', c_uint32), + ('accumulation_counter', c_uint64), + ('prochot_residency_acc', c_uint64), + ('ppt_residency_acc', c_uint64), + ('socket_thm_residency_acc', c_uint64), + ('vr_thm_residency_acc', c_uint64), + ('hbm_thm_residency_acc', c_uint64), + ('num_partition', c_uint16), + ('xcp_stats', xcp_stats_t * 8), + ('pcie_lc_perf_other_end_recovery', c_uint32), +] +amdsmi_gpu_metrics_t = rsmi_gpu_metrics_t diff --git a/projects/rocm-smi-lib/rocm_smi/example/rocm_smi_example.cc b/projects/rocm-smi-lib/rocm_smi/example/rocm_smi_example.cc index 84608e7259..93b004b083 100755 --- a/projects/rocm-smi-lib/rocm_smi/example/rocm_smi_example.cc +++ b/projects/rocm-smi-lib/rocm_smi/example/rocm_smi_example.cc @@ -731,30 +731,6 @@ template constexpr float convert_mw_to_w(T mw) { return static_cast(mw / 1000.0); } -template -auto print_error_or_value(rsmi_status_t status_code, const T& metric) { - if (status_code == rsmi_status_t::RSMI_STATUS_SUCCESS) { - if constexpr (std::is_array_v) { - auto idx = uint16_t(0); - auto str_values = std::string(); - const auto num_elems = static_cast(std::end(metric) - std::begin(metric)); - str_values = ("\n\t\t num of values: " + std::to_string(num_elems) + "\n"); - for (const auto& el : metric) { - str_values += "\t\t [" + std::to_string(idx) + "]: " + std::to_string(el) + "\n"; - ++idx; - } - return str_values; - } - else if constexpr ((std::is_same_v) || - (std::is_same_v) || - (std::is_same_v)) { - return std::to_string(metric); - } - } - else { - return ("\n\t\tStatus: [" + std::to_string(status_code) + "] " + "-> " + amd::smi::getRSMIStatusString(status_code)); - } -}; template std::string print_unsigned_int(T value) { @@ -860,8 +836,9 @@ int main() { // std::cout << "\n"; print_test_header("GPU METRICS: Using static struct (Backwards Compatibility) ", i); - print_function_header_with_rsmi_ret(ret, "rsmi_dev_gpu_metrics_info_get(" + std::to_string(i) + ", &gpu_metrics)"); - rsmi_dev_gpu_metrics_info_get(i, &gpu_metrics); + ret = rsmi_dev_gpu_metrics_info_get(i, &gpu_metrics); + print_function_header_with_rsmi_ret(ret, "rsmi_dev_gpu_metrics_info_get(" + + std::to_string(i) + ", &gpu_metrics)"); std::cout << "\t**.common_header.format_revision : " << print_unsigned_int(gpu_metrics.common_header.format_revision) << "\n"; @@ -960,6 +937,22 @@ int main() { << gpu_metrics.pcie_replay_count_acc << "\n"; std::cout << "\t**.pcie_replay_rover_count_acc : " << std::dec << gpu_metrics.pcie_replay_rover_count_acc << "\n"; + std::cout << "\t**.accumulation_counter : " << std::dec + << gpu_metrics.accumulation_counter << "\n"; + std::cout << "\t**.prochot_residency_acc : " << std::dec + << gpu_metrics.prochot_residency_acc << "\n"; + std::cout << "\t**.ppt_residency_acc : " << std::dec + << gpu_metrics.ppt_residency_acc << "\n"; + std::cout << "\t**.socket_thm_residency_acc : " << std::dec + << gpu_metrics.socket_thm_residency_acc << "\n"; + std::cout << "\t**.vr_thm_residency_acc : " << std::dec + << gpu_metrics.vr_thm_residency_acc << "\n"; + std::cout << "\t**.hbm_thm_residency_acc : " << std::dec + << gpu_metrics.hbm_thm_residency_acc << "\n"; + std::cout << "\t**.num_partition: " << std::dec + << gpu_metrics.num_partition << "\n"; + std::cout << "\t**.pcie_lc_perf_other_end_recovery: " + << gpu_metrics.pcie_lc_perf_other_end_recovery << "\n"; std::cout << "\t**.temperature_hbm[] : " << std::dec << "\n"; for (const auto& temp : gpu_metrics.temperature_hbm) { @@ -1001,23 +994,70 @@ int main() { std::cout << "\t -> " << std::dec << dclk << "\n"; } + std::cout << std::dec << "xcp_stats.gfx_busy_inst = \n"; + auto xcp = 0; + for (auto& row : gpu_metrics.xcp_stats) { + std::cout << "XCP[" << xcp << "] = " << "[ "; + std::copy(std::begin(row.gfx_busy_inst), + std::end(row.gfx_busy_inst), + amd::smi::make_ostream_joiner(&std::cout, ", ")); + std::cout << " ]\n"; + xcp++; + } + + xcp = 0; + std::cout << std::dec << "xcp_stats.jpeg_busy = \n"; + for (auto& row : gpu_metrics.xcp_stats) { + std::cout << "XCP[" << xcp << "] = " << "[ "; + std::copy(std::begin(row.jpeg_busy), + std::end(row.jpeg_busy), + amd::smi::make_ostream_joiner(&std::cout, ", ")); + std::cout << " ]\n"; + xcp++; + } + + xcp = 0; + std::cout << std::dec << "xcp_stats.vcn_busy = \n"; + for (auto& row : gpu_metrics.xcp_stats) { + std::cout << "XCP[" << xcp << "] = " << "[ "; + std::copy(std::begin(row.vcn_busy), + std::end(row.vcn_busy), + amd::smi::make_ostream_joiner(&std::cout, ", ")); + std::cout << " ]\n"; + xcp++; + } + + xcp = 0; + std::cout << std::dec << "xcp_stats.gfx_busy_acc = \n"; + for (auto& row : gpu_metrics.xcp_stats) { + std::cout << "XCP[" << xcp << "] = " << "[ "; + std::copy(std::begin(row.gfx_busy_acc), + std::end(row.gfx_busy_acc), + amd::smi::make_ostream_joiner(&std::cout, ", ")); + std::cout << " ]\n"; + xcp++; + } + std::cout << "\n"; std::cout << "\t ** -> Checking metrics with constant changes ** " << "\n"; constexpr uint16_t kMAX_ITER_TEST = 10; rsmi_gpu_metrics_t gpu_metrics_check; for (auto idx = uint16_t(1); idx <= kMAX_ITER_TEST; ++idx) { rsmi_dev_gpu_metrics_info_get(i, &gpu_metrics_check); - std::cout << "\t\t -> firmware_timestamp [" << idx << "/" << kMAX_ITER_TEST << "]: " << gpu_metrics_check.firmware_timestamp << "\n"; + std::cout << "\t\t -> firmware_timestamp [" << idx + << "/" << kMAX_ITER_TEST << "]: " << gpu_metrics_check.firmware_timestamp << "\n"; } std::cout << "\n"; for (auto idx = uint16_t(1); idx <= kMAX_ITER_TEST; ++idx) { rsmi_dev_gpu_metrics_info_get(i, &gpu_metrics_check); - std::cout << "\t\t -> system_clock_counter [" << idx << "/" << kMAX_ITER_TEST << "]: " << gpu_metrics_check.system_clock_counter << "\n"; + std::cout << "\t\t -> system_clock_counter [" << idx + << "/" << kMAX_ITER_TEST << "]: " << gpu_metrics_check.system_clock_counter << "\n"; } std::cout << "\n\n"; - std::cout << " ** Note: Values MAX'ed out (UINTX MAX are unsupported for the version in question) ** " << "\n"; + std::cout << " ** Note: Values MAX'ed out " + "(UINTX MAX are unsupported for the version in question) ** " << "\n"; std::cout << "\n\n"; @@ -1026,14 +1066,16 @@ int main() { ret = rsmi_dev_metrics_header_info_get(i, &header_values); std::cout << "\t[Metrics Header]" << "\n"; - std::cout << "\t -> format_revision : " << print_unsigned_int(header_values.format_revision) << "\n"; - std::cout << "\t -> content_revision : " << print_unsigned_int(header_values.content_revision) << "\n"; + std::cout << "\t -> format_revision : " + << print_unsigned_int(header_values.format_revision) << "\n"; + std::cout << "\t -> content_revision : " + << print_unsigned_int(header_values.content_revision) << "\n"; std::cout << "\t--------------------" << "\n"; std::cout << "\n"; std::cout << "\t[XCD CounterVoltage]" << "\n"; ret = rsmi_dev_metrics_xcd_counter_get(i, &val_ui16); - std::cout << "\t -> xcd_counter(): " << print_error_or_value(ret, val_ui16) << "\n"; + std::cout << "\t -> xcd_counter(): " << val_ui16; std::cout << "\n\n"; ret = rsmi_dev_perf_level_get(i, &pfl); @@ -1041,8 +1083,12 @@ int main() { std::cout << "\t**Performance Level:" << perf_level_string(pfl) << "\n"; ret = rsmi_dev_overdrive_level_get(i, &val_ui32); - CHK_AND_PRINT_RSMI_ERR_RET(ret) - std::cout << "\t**OverDrive Level:" << val_ui32 << "\n"; + std::cout << "\t**OverDrive Level: "; + if (ret == RSMI_STATUS_SUCCESS) { + std::cout << val_ui32 << "\n"; + } else { + CHK_RSMI_NOT_SUPPORTED_OR_UNEXPECTED_DATA_RET(ret) + } print_test_header("GPU Clocks", i); for (int clkType = static_cast(RSMI_CLK_TYPE_SYS); @@ -1159,9 +1205,6 @@ int main() { } for (uint32_t i = 0; i < num_monitor_devs; ++i) { - ret = test_set_overdrive(i); - CHK_AND_PRINT_RSMI_ERR_RET(ret) - ret = test_set_perf_level(i); CHK_AND_PRINT_RSMI_ERR_RET(ret) @@ -1182,6 +1225,9 @@ int main() { ret = test_set_memory_partition(i); CHK_AND_PRINT_RSMI_ERR_RET(ret) + + ret = test_set_overdrive(i); + CHK_RSMI_NOT_SUPPORTED_RET(ret) } return 0; diff --git a/projects/rocm-smi-lib/src/rocm_smi.cc b/projects/rocm-smi-lib/src/rocm_smi.cc index c728ee9b07..1baef2def5 100755 --- a/projects/rocm-smi-lib/src/rocm_smi.cc +++ b/projects/rocm-smi-lib/src/rocm_smi.cc @@ -5846,6 +5846,9 @@ rsmi_dev_metrics_xcd_counter_get(uint32_t dv_ind, uint16_t* xcd_counter_value) auto status_code = rsmi_dev_gpu_metrics_info_get(dv_ind, &gpu_metrics); if (status_code == rsmi_status_t::RSMI_STATUS_SUCCESS) { for (const auto& gfxclk : gpu_metrics.current_gfxclks) { + if (gfxclk == UINT16_MAX) { + break; + } if ((gfxclk != 0) && (gfxclk != UINT16_MAX)) { xcd_counter++; } diff --git a/projects/rocm-smi-lib/src/rocm_smi_device.cc b/projects/rocm-smi-lib/src/rocm_smi_device.cc index 526600d505..8f2f170469 100755 --- a/projects/rocm-smi-lib/src/rocm_smi_device.cc +++ b/projects/rocm-smi-lib/src/rocm_smi_device.cc @@ -964,15 +964,17 @@ int Device::readDevInfoBinary(DevInfoTypes type, std::size_t b_size, LOG_ERROR(ss); return ENOENT; } - ss << "Successfully read DevInfoBinary for DevInfoType (" - << devInfoTypesStrings.at(type) << ") - SYSFS (" - << sysfs_path << "), returning binaryData = " << p_binary_data - << "; byte_size = " << std::dec << static_cast(b_size); + if (ROCmLogging::Logger::getInstance()->isLoggerEnabled()) { + ss << "Successfully read DevInfoBinary for DevInfoType (" + << devInfoTypesStrings.at(type) << ") - SYSFS (" + << sysfs_path << "), returning binaryData = " << p_binary_data + << "; byte_size = " << std::dec << static_cast(b_size); - std::string metricDescription = "AMD SMI GPU METRICS (16-byte width), " + std::string metricDescription = "AMD SMI GPU METRICS (16-byte width), " + sysfs_path; - logHexDump(metricDescription.c_str(), p_binary_data, b_size, 16); - LOG_INFO(ss); + logHexDump(metricDescription.c_str(), p_binary_data, b_size, 16); + LOG_INFO(ss); + } return 0; } diff --git a/projects/rocm-smi-lib/src/rocm_smi_gpu_metrics.cc b/projects/rocm-smi-lib/src/rocm_smi_gpu_metrics.cc index 08019ff2b7..c34fa09040 100755 --- a/projects/rocm-smi-lib/src/rocm_smi_gpu_metrics.cc +++ b/projects/rocm-smi-lib/src/rocm_smi_gpu_metrics.cc @@ -3,7 +3,7 @@ * The University of Illinois/NCSA * Open Source License (NCSA) * - * Copyright (c) 2017-2023, Advanced Micro Devices, Inc. + * Copyright (c) 2017-2024, Advanced Micro Devices, Inc. * All rights reserved. * * Developed by: @@ -156,6 +156,7 @@ std::string stringfy_metric_header_version(const AMDGpuMetricsHeader_v1_t& metri // version 1.3: 259 // version 1.4: 260 // version 1.5: 261 +// version 1.6: 262 // const AMDGpuMetricVersionTranslationTbl_t amdgpu_metric_version_translation_table { @@ -164,6 +165,7 @@ const AMDGpuMetricVersionTranslationTbl_t amdgpu_metric_version_translation_tabl {join_metrics_version(1, 3), AMDGpuMetricVersionFlags_t::kGpuMetricV13}, {join_metrics_version(1, 4), AMDGpuMetricVersionFlags_t::kGpuMetricV14}, {join_metrics_version(1, 5), AMDGpuMetricVersionFlags_t::kGpuMetricV15}, + {join_metrics_version(1, 6), AMDGpuMetricVersionFlags_t::kGpuMetricV16}, }; /** @@ -183,10 +185,12 @@ const AMDGpuMetricsClassIdTranslationTbl_t amdgpu_metrics_class_id_translation_t {AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed, "Link/Bandwidth/Speed"}, {AMDGpuMetricsClassId_t::kGpuMetricVoltage, "Voltage"}, {AMDGpuMetricsClassId_t::kGpuMetricTimestamp, "Timestamp"}, + {AMDGpuMetricsClassId_t::kGpuMetricThrottleResidency, "Throttler Residency"}, + {AMDGpuMetricsClassId_t::kGpuMetricPartition, "Partition Number"}, + {AMDGpuMetricsClassId_t::kGpuMetricXcpStats, "XCP Stats"}, }; -const AMDGpuMetricsUnitTypeTranslationTbl_t amdgpu_metrics_unit_type_translation_table -{ +const AMDGpuMetricsUnitTypeTranslationTbl_t amdgpu_metrics_unit_type_translation_table { // kGpuMetricTemperature counters {AMDGpuMetricsUnitType_t::kMetricTempEdge, "TempEdge"}, {AMDGpuMetricsUnitType_t::kMetricTempHotspot, "TempHotspot"}, @@ -261,20 +265,40 @@ const AMDGpuMetricsUnitTypeTranslationTbl_t amdgpu_metrics_unit_type_translation // kGpuMetricTimestamp counters {AMDGpuMetricsUnitType_t::kMetricTSClockCounter, "TSClockCounter"}, {AMDGpuMetricsUnitType_t::kMetricTSFirmware, "TSFirmware"}, + + // kGpuMetricThrottleResidency counters + {AMDGpuMetricsUnitType_t::kMetricAccumulationCounter, "AccumulationCounter"}, /* v1.6 */ + {AMDGpuMetricsUnitType_t::kMetricProchotResidencyAccumulator, "ProchotResidencyAccumulator"}, /* v1.6 */ + {AMDGpuMetricsUnitType_t::kMetricPPTResidencyAccumulator, "PPTResidencyAccumulator"}, /* v1.6 */ + {AMDGpuMetricsUnitType_t::kMetricSocketThmResidencyAccumulator, "SocketThmResidencyAccumulator"}, /* v1.6 */ + {AMDGpuMetricsUnitType_t::kMetricVRThmResidencyAccumulator, "VRThmResidencyAccumulator"}, /* v1.6 */ + {AMDGpuMetricsUnitType_t::kMetricHBMThmResidencyAccumulator, "HBMThmResidencyAccumulator"}, /* v1.6 */ + + // kGpuMetricPartition + {AMDGpuMetricsUnitType_t::kGpuMetricNumPartition, "numPartition"}, /* v1.6 */ + + // kGpuMetricXcpStats + {AMDGpuMetricsUnitType_t::kMetricGfxBusyInst, "GfxBusyInst"}, /* v1.6 */ + {AMDGpuMetricsUnitType_t::kMetricJpegBusy, "JpegBusy"}, /* v1.6 */ + {AMDGpuMetricsUnitType_t::kMetricVcnBusy, "VcnBusy"}, /* v1.6 */ + {AMDGpuMetricsUnitType_t::kMetricGfxBusyAcc, "GfxBusyAcc"}, /* v1.6 */ + + // kGpuMetricLinkWidthSpeed + {AMDGpuMetricsUnitType_t::kMetricPcieLCPerfOtherEndRecov, "PcieLCPerfOtherEndRecov"}, /* v1.6 */ }; AMDGpuMetricVersionFlags_t translate_header_to_flag_version(const AMDGpuMetricsHeader_v1_t& metrics_header) { - std::ostringstream ostrstream; + std::ostringstream ss; auto version_id(AMDGpuMetricVersionFlags_t::kGpuMetricNone); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); const auto flag_version = join_metrics_version(metrics_header); if (amdgpu_metric_version_translation_table.find(flag_version) != amdgpu_metric_version_translation_table.end()) { version_id = amdgpu_metric_version_translation_table.at(flag_version); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Translation Tbl: " << flag_version @@ -282,11 +306,11 @@ AMDGpuMetricVersionFlags_t translate_header_to_flag_version(const AMDGpuMetricsH << " | Returning = " << static_cast(version_id) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return version_id; } - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Translation Tbl: " << flag_version @@ -294,40 +318,40 @@ AMDGpuMetricVersionFlags_t translate_header_to_flag_version(const AMDGpuMetricsH << " | Returning = " << static_cast(version_id) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return version_id; } uint16_t translate_flag_to_metric_version(AMDGpuMetricVersionFlags_t version_flag) { - std::ostringstream ostrstream; + std::ostringstream ss; auto version_id = uint16_t(0); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); for (const auto& [key, value] : amdgpu_metric_version_translation_table) { if (value == version_flag) { version_id = key; - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Version Flag: " << static_cast(version_flag) << " | Unified Version: " << version_id << " | Str. Version: " << stringfy_metric_header_version(disjoin_metrics_version(version_id)) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return version_id; } } - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Version Flag: " << static_cast(version_flag) << " | Unified Version: " << version_id << " | Str. Version: " << stringfy_metric_header_version(disjoin_metrics_version(version_id)) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return version_id; } @@ -348,37 +372,38 @@ AMDGpuMetricFactories_t amd_gpu_metrics_factory_table {AMDGpuMetricVersionFlags_t::kGpuMetricV13, std::make_shared(GpuMetricsBase_v13_t{})}, {AMDGpuMetricVersionFlags_t::kGpuMetricV14, std::make_shared(GpuMetricsBase_v14_t{})}, {AMDGpuMetricVersionFlags_t::kGpuMetricV15, std::make_shared(GpuMetricsBase_v15_t{})}, + {AMDGpuMetricVersionFlags_t::kGpuMetricV16, std::make_shared(GpuMetricsBase_v16_t{})}, }; GpuMetricsBasePtr amdgpu_metrics_factory(AMDGpuMetricVersionFlags_t gpu_metric_version) { - std::ostringstream ostrstream; - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + std::ostringstream ss; + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); auto contains = [](const AMDGpuMetricVersionFlags_t metric_version) { return (amd_gpu_metrics_factory_table.find(metric_version) != amd_gpu_metrics_factory_table.end()); }; if (contains(gpu_metric_version)) { - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Factory Version: " << static_cast(gpu_metric_version) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return (amd_gpu_metrics_factory_table.at(gpu_metric_version)); } - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Factory Version: " << static_cast(gpu_metric_version) << " | Returning = " << "No object from factory." << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return nullptr; } @@ -448,8 +473,7 @@ AMDGpuDynamicMetricTblValues_t format_metric_row(const T& metric, const std::str auto value = uint64_t(0); if constexpr (std::is_array_v) { value = (metric[idx]); - } - else { + } else { value = (metric); } @@ -467,17 +491,444 @@ AMDGpuDynamicMetricTblValues_t format_metric_row(const T& metric, const std::str return multi_values; } +void GpuMetricsBase_v16_t::dump_internal_metrics_table() +{ + std::ostringstream ss; + auto idx = uint64_t(0); + auto idy = uint64_t(0); + std::cout << __PRETTY_FUNCTION__ << " | ======= start ======= \n"; + ss << __PRETTY_FUNCTION__ + << " | ======= DEBUG ======= " + << " | Metric Version: " + << stringfy_metric_header_version(m_gpu_metrics_tbl.m_common_header) + << " | Size: " + << print_unsigned_int(m_gpu_metrics_tbl.m_common_header.m_structure_size) + << " |" + << "\n"; + ss << " temperature_hotspot: " << m_gpu_metrics_tbl.m_temperature_hotspot << "\n" + << " temperature_mem: " << m_gpu_metrics_tbl.m_temperature_mem << "\n" + << " temperature_vrsoc: " << m_gpu_metrics_tbl.m_temperature_vrsoc << "\n" + << " current_socket_power: " << m_gpu_metrics_tbl.m_current_socket_power << "\n" + << " average_gfx_activity: " << m_gpu_metrics_tbl.m_average_gfx_activity << "\n" + << " average_umc_activity: " << m_gpu_metrics_tbl.m_average_umc_activity << "\n"; + + ss << " energy_accumulator: " << m_gpu_metrics_tbl.m_energy_accumulator << "\n" + << " system_clock_counter: " << m_gpu_metrics_tbl.m_system_clock_counter << "\n" + << " accumulation_counter: " << m_gpu_metrics_tbl.m_accumulation_counter << "\n" + << " prochot_residency_acc: " << m_gpu_metrics_tbl.m_prochot_residency_acc << "\n" + << " ppt_residency_acc: " << m_gpu_metrics_tbl.m_ppt_residency_acc << "\n" + << " socket_thm_residency_acc: " << m_gpu_metrics_tbl.m_socket_thm_residency_acc << "\n" + << " vr_thm_residency_acc: " << m_gpu_metrics_tbl.m_vr_thm_residency_acc << "\n" + << " hbm_thm_residency_acc: " << m_gpu_metrics_tbl.m_hbm_thm_residency_acc << "\n" + << " average_gfx_activity: " << m_gpu_metrics_tbl.m_average_gfx_activity << "\n" + << " average_umc_activity: " << m_gpu_metrics_tbl.m_average_umc_activity << "\n" + << " gfxclk_lock_status: " << m_gpu_metrics_tbl.m_gfxclk_lock_status << "\n" + << " pcie_link_width: " << m_gpu_metrics_tbl.m_pcie_link_width << "\n" + << " pcie_link_speed: " << m_gpu_metrics_tbl.m_pcie_link_speed << "\n" + << " xgmi_link_width: " << m_gpu_metrics_tbl.m_xgmi_link_width << "\n" + << " xgmi_link_speed: " << m_gpu_metrics_tbl.m_xgmi_link_speed << "\n" + << " gfx_activity_acc: " << m_gpu_metrics_tbl.m_gfx_activity_acc << "\n" + << " mem_activity_acc: " << m_gpu_metrics_tbl.m_mem_activity_acc << "\n" + << " pcie_bandwidth_acc: " << m_gpu_metrics_tbl.m_pcie_bandwidth_acc << "\n" + << " pcie_bandwidth_inst: " << m_gpu_metrics_tbl.m_pcie_bandwidth_inst << "\n" + << " pcie_l0_to_recov_count_acc: " << m_gpu_metrics_tbl.m_pcie_l0_to_recov_count_acc << "\n" + << " pcie_replay_count_acc: " << m_gpu_metrics_tbl.m_pcie_replay_count_acc << "\n" + << " pcie_replay_rover_count_acc: " << m_gpu_metrics_tbl.m_pcie_replay_rover_count_acc << "\n" + << " pcie_nak_sent_count_acc: " << m_gpu_metrics_tbl.m_pcie_nak_sent_count_acc << "\n" + << " pcie_nak_rcvd_count_acc: " << m_gpu_metrics_tbl.m_pcie_nak_rcvd_count_acc << "\n" + << " firmware_timestamp: " << m_gpu_metrics_tbl.m_firmware_timestamp << "\n" + << " current_uclk: " << m_gpu_metrics_tbl.m_current_uclk << "\n" + << " num_partition: " << m_gpu_metrics_tbl.m_num_partition << "\n" + << " pcie_lc_perf_other_end_recovery: " + << m_gpu_metrics_tbl.m_pcie_lc_perf_other_end_recovery << "\n"; + idx = 0; + for (const auto& temp : m_gpu_metrics_tbl.m_xgmi_read_data_acc) { + ss << "\t [" << idx << "]: " << temp << "\n"; + ++idx; + } + + ss << " xgmi_write_data_acc: " << "\n"; + idx = 0; + for (const auto& temp : m_gpu_metrics_tbl.m_xgmi_write_data_acc) { + ss << "\t [" << idx << "]: " << temp << "\n"; + ++idx; + } + + ss << " current_gfxclk: " << "\n"; + idx = 0; + for (const auto& temp : m_gpu_metrics_tbl.m_current_gfxclk) { + ss << "\t [" << idx << "]: " << temp << "\n"; + ++idx; + } + + ss << " current_socclk: " << "\n"; + idx = 0; + for (const auto& temp : m_gpu_metrics_tbl.m_current_socclk) { + ss << "\t [" << idx << "]: " << temp << "\n"; + ++idx; + } + + ss << " current_vclk0: " << "\n"; + idx = 0; + for (const auto& temp : m_gpu_metrics_tbl.m_current_vclk0) { + ss << "\t [" << idx << "]: " << temp << "\n"; + ++idx; + } + + ss << " current_dclk0: " << "\n"; + idx = 0; + for (const auto& temp : m_gpu_metrics_tbl.m_current_dclk0) { + ss << "\t [" << idx << "]: " << temp << "\n"; + ++idx; + } + + idx = 0; + idy = 0; + ss << " xcp_stats.gfx_busy_inst: " << "\n"; + for (auto& row : m_gpu_metrics_tbl.m_xcp_stats) { + if (idx == 0) { + ss << "\t [ "; + } + for (auto& col : row.gfx_busy_inst) { + ss << "\t [" << idx << "] [" << idy << "]: " << col; + if (idy + 1 != (std::end(row.gfx_busy_inst) - std::end(row.gfx_busy_inst) - 1)) { + ss << ", "; + } + if (idx + 1 != + (std::end(m_gpu_metrics_tbl.m_xcp_stats) - std::end(m_gpu_metrics_tbl.m_xcp_stats) - 1)) { + ss << "\n"; + } else { + ss << "]\n"; + } + idy++; + } + idx++; + } + + idx = 0; + idy = 0; + ss << " xcp_stats.vcn_busy: " << "\n"; + for (auto& row : m_gpu_metrics_tbl.m_xcp_stats) { + if (idx == 0) { + ss << "\t [ "; + } + for (auto& col : row.vcn_busy) { + ss << "\t [" << idx << "] [" << idy << "]: " << col; + if (idy + 1 != (std::end(row.vcn_busy) - std::end(row.vcn_busy) - 1)) { + ss << ", "; + } + if (idx + 1 != + (std::end(m_gpu_metrics_tbl.m_xcp_stats) - std::end(m_gpu_metrics_tbl.m_xcp_stats) - 1)) { + ss << "\n"; + } else { + ss << "]\n"; + } + idy++; + } + idx++; + } + + idx = 0; + idy = 0; + ss << " xcp_stats.jpeg_busy: " << "\n"; + for (auto& row : m_gpu_metrics_tbl.m_xcp_stats) { + if (idx == 0) { + ss << "\t [ "; + } + for (auto& col : row.jpeg_busy) { + ss << "\t [" << idx << "] [" << idy << "]: " << col; + if (idy + 1 != (std::end(row.jpeg_busy) - std::end(row.jpeg_busy) - 1)) { + ss << ", "; + } + if (idx + 1 != + (std::end(m_gpu_metrics_tbl.m_xcp_stats) - std::end(m_gpu_metrics_tbl.m_xcp_stats) - 1)) { + ss << "\n"; + } else { + ss << "]\n"; + } + idy++; + } + idx++; + } + + idx = 0; + idy = 0; + ss << " xcp_stats.gfx_busy_acc: " << "\n"; + for (auto& row : m_gpu_metrics_tbl.m_xcp_stats) { + if (idx == 0) { + ss << "\t [ "; + } + for (auto& col : row.gfx_busy_acc) { + ss << "\t [" << idx << "] [" << idy << "]: " << col; + if (idy + 1 != (std::end(row.gfx_busy_acc) - std::end(row.gfx_busy_acc) - 1)) { + ss << ", "; + } + if (idx + 1 != + (std::end(m_gpu_metrics_tbl.m_xcp_stats) - std::end(m_gpu_metrics_tbl.m_xcp_stats) - 1)) { + ss << "\n"; + } else { + ss << "]\n"; + } + idy++; + } + idx++; + } + + LOG_DEBUG(ss); +} + +rsmi_status_t GpuMetricsBase_v16_t::populate_metrics_dynamic_tbl() { + std::ostringstream ss; + auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); + + if (!m_metrics_dynamic_tbl.empty()) { + m_metrics_dynamic_tbl.clear(); + } + + // + // Note: Any metric treatment/changes (if any) should happen before they + // get written to internal/external tables. + // + auto run_metric_adjustments_v16 = [&]() { + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + const auto gpu_metrics_version = + translate_flag_to_metric_version(get_gpu_metrics_version_used()); + ss << __PRETTY_FUNCTION__ + << " | ======= info ======= " + << " | Applying adjustments " + << " | Metric Version: " << stringfy_metric_header_version( + disjoin_metrics_version(gpu_metrics_version)) + << " |"; + LOG_TRACE(ss); + + // firmware_timestamp is at 10ns resolution + ss << __PRETTY_FUNCTION__ + << " | ======= Changes ======= " + << " | {m_firmware_timestamp} from: " << m_gpu_metrics_tbl.m_firmware_timestamp + << " to: " << (m_gpu_metrics_tbl.m_firmware_timestamp * 10); + m_gpu_metrics_tbl.m_firmware_timestamp = (m_gpu_metrics_tbl.m_firmware_timestamp * 10); + LOG_DEBUG(ss); + }; + + // Adjustments/Changes specific to this version + run_metric_adjustments_v16(); + + // Temperature Info + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricTemperature] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricTempHotspot, + format_metric_row(m_gpu_metrics_tbl.m_temperature_hotspot, + "temperature_hotspot"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricTemperature] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricTempMem, + format_metric_row(m_gpu_metrics_tbl.m_temperature_mem, + "temperature_mem"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricTemperature] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricTempVrSoc, + format_metric_row(m_gpu_metrics_tbl.m_temperature_vrsoc, + "temperature_vrsoc"))); + + // Power/Energy Info + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricPowerEnergy] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricCurrSocketPower, + format_metric_row(m_gpu_metrics_tbl.m_current_socket_power, + "curr_socket_power"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricPowerEnergy] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricEnergyAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_energy_accumulator, + "energy_acc"))); + + // Utilization Info + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricUtilization] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricAvgGfxActivity, + format_metric_row(m_gpu_metrics_tbl.m_average_gfx_activity, + "average_gfx_activity"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricUtilization] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricAvgUmcActivity, + format_metric_row(m_gpu_metrics_tbl.m_average_umc_activity, + "average_umc_activity"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricUtilization] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricGfxActivityAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_gfx_activity_acc, + "gfx_activity_acc"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricUtilization] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricMemActivityAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_mem_activity_acc, + "mem_activity_acc"))); + + // Timestamp Info + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricTimestamp] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricTSFirmware, + format_metric_row(m_gpu_metrics_tbl.m_firmware_timestamp, + "firmware_timestamp"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricTimestamp] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricTSClockCounter, + format_metric_row(m_gpu_metrics_tbl.m_system_clock_counter, + "system_clock_counter"))); + + + // GfxLock Info + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricGfxClkLockStatus] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricGfxClkLockStatus, + format_metric_row(m_gpu_metrics_tbl.m_gfxclk_lock_status, + "gfxclk_lock_status"))); + + // Link/Width/Speed Info + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricPcieLinkWidth, + format_metric_row(m_gpu_metrics_tbl.m_pcie_link_width, + "pcie_link_width"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricPcieLinkSpeed, + format_metric_row(m_gpu_metrics_tbl.m_pcie_link_speed, + "pcie_link_speed"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricXgmiLinkWidth, + format_metric_row(m_gpu_metrics_tbl.m_xgmi_link_width, + "xgmi_link_width"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricXgmiLinkSpeed, + format_metric_row(m_gpu_metrics_tbl.m_xgmi_link_speed, + "xgmi_link_speed"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricPcieBandwidthAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_pcie_bandwidth_acc, + "pcie_bandwidth_acc"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricPcieBandwidthInst, + format_metric_row(m_gpu_metrics_tbl.m_pcie_bandwidth_inst, + "pcie_bandwidth_inst"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricPcieL0RecovCountAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_pcie_l0_to_recov_count_acc, + "pcie_l0_recov_count_acc"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricPcieReplayCountAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_pcie_replay_count_acc, + "pcie_replay_count_acc"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricPcieReplayRollOverCountAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_pcie_replay_rover_count_acc, + "pcie_replay_rollover_count_acc"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricPcieNakSentCountAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_pcie_nak_sent_count_acc, + "pcie_nak_sent_count_acc"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricPcieNakReceivedCountAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_pcie_nak_rcvd_count_acc, + "pcie_nak_rcvd_count_acc"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricXgmiReadDataAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_xgmi_read_data_acc, + "[xgmi_read_data_acc]"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricXgmiWriteDataAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_xgmi_write_data_acc, + "[xgmi_write_data_acc]"))); + + // CurrentClock Info + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricCurrentClock] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricCurrGfxClock, + format_metric_row(m_gpu_metrics_tbl.m_current_gfxclk, + "[current_gfxclk]"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricCurrentClock] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricCurrSocClock, + format_metric_row(m_gpu_metrics_tbl.m_current_socclk, + "[current_socclk]"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricCurrentClock] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricCurrVClock0, + format_metric_row(m_gpu_metrics_tbl.m_current_vclk0, + "[current_vclk0]"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricCurrentClock] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricCurrDClock0, + format_metric_row(m_gpu_metrics_tbl.m_current_dclk0, + "[current_dclk0]"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricCurrentClock] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricCurrUClock, + format_metric_row(m_gpu_metrics_tbl.m_current_uclk, + "current_uclk"))); + + /* Accumulation cycle counter */ + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricThrottleResidency] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricAccumulationCounter, + format_metric_row(m_gpu_metrics_tbl.m_accumulation_counter, + "accumulation_counter"))); + + /* Accumulated throttler residencies */ + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricThrottleResidency] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricProchotResidencyAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_prochot_residency_acc, + "prochot_residency_acc"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricThrottleResidency] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricPPTResidencyAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_ppt_residency_acc, + "ppt_residency_acc"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricThrottleResidency] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricSocketThmResidencyAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_socket_thm_residency_acc, + "socket_thm_residency_acc"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricThrottleResidency] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricVRThmResidencyAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_vr_thm_residency_acc, + "vr_thm_residency_acc"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricThrottleResidency] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricHBMThmResidencyAccumulator, + format_metric_row(m_gpu_metrics_tbl.m_hbm_thm_residency_acc, + "hbm_thm_residency_acc"))); + + /* Partition info */ + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricPartition] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kGpuMetricNumPartition, + format_metric_row(m_gpu_metrics_tbl.m_num_partition, + "num_partition"))); + + /* xcp_stats info */ + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricXcpStats] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricGfxBusyInst, + format_metric_row(m_gpu_metrics_tbl.m_xcp_stats->gfx_busy_inst, + "xcp_stats->gfx_busy_inst"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricXcpStats] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricVcnBusy, + format_metric_row(m_gpu_metrics_tbl.m_xcp_stats->vcn_busy, + "xcp_stats->vcn_busy"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricXcpStats] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricJpegBusy, + format_metric_row(m_gpu_metrics_tbl.m_xcp_stats->jpeg_busy, + "xcp_stats->jpeg_busy"))); + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricXcpStats] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricGfxBusyAcc, + format_metric_row(m_gpu_metrics_tbl.m_xcp_stats->gfx_busy_acc, + "xcp_stats->gfx_busy_acc"))); + + /* PCIE other end recovery counter info */ + m_metrics_dynamic_tbl[AMDGpuMetricsClassId_t::kGpuMetricLinkWidthSpeed] + .insert(std::make_pair(AMDGpuMetricsUnitType_t::kMetricPcieLCPerfOtherEndRecov, + format_metric_row(m_gpu_metrics_tbl.m_pcie_lc_perf_other_end_recovery, + "pcie_lc_perf_other_end_recovery"))); + + ss << __PRETTY_FUNCTION__ + << " | ======= end ======= " + << " | Success " + << " | Returning = " << getRSMIStatusString(status_code) + << " |"; + LOG_TRACE(ss); + + return status_code; +} + void GpuMetricsBase_v15_t::dump_internal_metrics_table() { - std::ostringstream ostrstream; + std::ostringstream ss; std::cout << __PRETTY_FUNCTION__ << " | ======= start ======= \n"; - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= DEBUG ======= " << " | Metric Version: " << stringfy_metric_header_version(m_gpu_metrics_tbl.m_common_header) << " | Size: " << print_unsigned_int(m_gpu_metrics_tbl.m_common_header.m_structure_size) << " |" << "\n"; - ostrstream << " temperature_hotspot: " << m_gpu_metrics_tbl.m_temperature_hotspot << "\n" + ss << " temperature_hotspot: " << m_gpu_metrics_tbl.m_temperature_hotspot << "\n" << " temperature_mem: " << m_gpu_metrics_tbl.m_temperature_mem << "\n" << " temperature_vrsoc: " << m_gpu_metrics_tbl.m_temperature_vrsoc << "\n" @@ -486,21 +937,21 @@ void GpuMetricsBase_v15_t::dump_internal_metrics_table() << " average_gfx_activity: " << m_gpu_metrics_tbl.m_average_gfx_activity << "\n" << " average_umc_activity: " << m_gpu_metrics_tbl.m_average_umc_activity << "\n"; - ostrstream << " vcn_activity: " << "\n"; + ss << " vcn_activity: " << "\n"; auto idx = uint64_t(0); for (const auto& temp : m_gpu_metrics_tbl.m_vcn_activity) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " jpeg_activity: " << "\n"; + ss << " jpeg_activity: " << "\n"; idx = 0; for (const auto& temp : m_gpu_metrics_tbl.m_jpeg_activity) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " energy_accumulator: " << m_gpu_metrics_tbl.m_energy_accumulator << "\n" + ss << " energy_accumulator: " << m_gpu_metrics_tbl.m_energy_accumulator << "\n" << " system_clock_counter: " << m_gpu_metrics_tbl.m_system_clock_counter << "\n" << " throttle_status: " << m_gpu_metrics_tbl.m_throttle_status << "\n" @@ -527,83 +978,86 @@ void GpuMetricsBase_v15_t::dump_internal_metrics_table() << " pcie_nak_sent_count_acc: " << m_gpu_metrics_tbl.m_pcie_nak_sent_count_acc << "\n" << " pcie_nak_rcvd_count_acc: " << m_gpu_metrics_tbl.m_pcie_nak_rcvd_count_acc << "\n"; - ostrstream << " xgmi_read_data_acc: " << "\n"; + ss << " xgmi_read_data_acc: " << "\n"; idx = 0; for (const auto& temp : m_gpu_metrics_tbl.m_xgmi_read_data_acc) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " xgmi_write_data_acc: " << "\n"; + ss << " xgmi_write_data_acc: " << "\n"; idx = 0; for (const auto& temp : m_gpu_metrics_tbl.m_xgmi_write_data_acc) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " firmware_timestamp: " << m_gpu_metrics_tbl.m_firmware_timestamp << "\n"; + ss << " firmware_timestamp: " << m_gpu_metrics_tbl.m_firmware_timestamp << "\n"; - ostrstream << " current_gfxclk: " << "\n"; + ss << " current_gfxclk: " << "\n"; idx = 0; for (const auto& temp : m_gpu_metrics_tbl.m_current_gfxclk) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " current_socclk: " << "\n"; + ss << " current_socclk: " << "\n"; idx = 0; for (const auto& temp : m_gpu_metrics_tbl.m_current_socclk) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " current_vclk0: " << "\n"; + ss << " current_vclk0: " << "\n"; idx = 0; for (const auto& temp : m_gpu_metrics_tbl.m_current_vclk0) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " current_dclk0: " << "\n"; + ss << " current_dclk0: " << "\n"; idx = 0; for (const auto& temp : m_gpu_metrics_tbl.m_current_dclk0) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " padding: " << m_gpu_metrics_tbl.m_padding << "\n"; - LOG_DEBUG(ostrstream); + ss << " padding: " << m_gpu_metrics_tbl.m_padding << "\n"; + LOG_DEBUG(ss); } -rsmi_status_t GpuMetricsBase_v15_t::populate_metrics_dynamic_tbl() -{ - std::ostringstream ostrstream; +rsmi_status_t GpuMetricsBase_v15_t::populate_metrics_dynamic_tbl() { + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); + + if (!m_metrics_dynamic_tbl.empty()) { + m_metrics_dynamic_tbl.clear(); + } // // Note: Any metric treatment/changes (if any) should happen before they // get written to internal/external tables. // auto run_metric_adjustments_v15 = [&]() { - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; const auto gpu_metrics_version = translate_flag_to_metric_version(get_gpu_metrics_version_used()); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= info ======= " << " | Applying adjustments " << " | Metric Version: " << stringfy_metric_header_version( disjoin_metrics_version(gpu_metrics_version)) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); // firmware_timestamp is at 10ns resolution - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= Changes ======= " << " | {m_firmware_timestamp} from: " << m_gpu_metrics_tbl.m_firmware_timestamp << " to: " << (m_gpu_metrics_tbl.m_firmware_timestamp * 10); m_gpu_metrics_tbl.m_firmware_timestamp = (m_gpu_metrics_tbl.m_firmware_timestamp * 10); - LOG_DEBUG(ostrstream); + LOG_DEBUG(ss); }; @@ -791,12 +1245,12 @@ rsmi_status_t GpuMetricsBase_v15_t::populate_metrics_dynamic_tbl() "current_uclk")) ); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return status_code; } @@ -804,15 +1258,15 @@ rsmi_status_t GpuMetricsBase_v15_t::populate_metrics_dynamic_tbl() void GpuMetricsBase_v14_t::dump_internal_metrics_table() { - std::ostringstream ostrstream; + std::ostringstream ss; std::cout << __PRETTY_FUNCTION__ << " | ======= start ======= \n"; - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= DEBUG ======= " << " | Metric Version: " << stringfy_metric_header_version(m_gpu_metrics_tbl.m_common_header) << " | Size: " << print_unsigned_int(m_gpu_metrics_tbl.m_common_header.m_structure_size) << " |" << "\n"; - ostrstream << " temperature_hotspot: " << m_gpu_metrics_tbl.m_temperature_hotspot << "\n" + ss << " temperature_hotspot: " << m_gpu_metrics_tbl.m_temperature_hotspot << "\n" << " temperature_mem: " << m_gpu_metrics_tbl.m_temperature_mem << "\n" << " temperature_vrsoc: " << m_gpu_metrics_tbl.m_temperature_vrsoc << "\n" @@ -821,14 +1275,14 @@ void GpuMetricsBase_v14_t::dump_internal_metrics_table() << " average_gfx_activity: " << m_gpu_metrics_tbl.m_average_gfx_activity << "\n" << " average_umc_activity: " << m_gpu_metrics_tbl.m_average_umc_activity << "\n"; - ostrstream << " vcn_activity: " << "\n"; + ss << " vcn_activity: " << "\n"; auto idx = uint64_t(0); for (const auto& temp : m_gpu_metrics_tbl.m_vcn_activity) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " energy_accumulator: " << m_gpu_metrics_tbl.m_energy_accumulator << "\n" + ss << " energy_accumulator: " << m_gpu_metrics_tbl.m_energy_accumulator << "\n" << " system_clock_counter: " << m_gpu_metrics_tbl.m_system_clock_counter << "\n" << " throttle_status: " << m_gpu_metrics_tbl.m_throttle_status << "\n" @@ -853,83 +1307,86 @@ void GpuMetricsBase_v14_t::dump_internal_metrics_table() << " pcie_replay_count_acc: " << m_gpu_metrics_tbl.m_pcie_replay_count_acc << "\n" << " pcie_replay_rover_count_acc: " << m_gpu_metrics_tbl.m_pcie_replay_rover_count_acc << "\n"; - ostrstream << " xgmi_read_data_acc: " << "\n"; + ss << " xgmi_read_data_acc: " << "\n"; idx = 0; for (const auto& temp : m_gpu_metrics_tbl.m_xgmi_read_data_acc) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " xgmi_write_data_acc: " << "\n"; + ss << " xgmi_write_data_acc: " << "\n"; idx = 0; for (const auto& temp : m_gpu_metrics_tbl.m_xgmi_write_data_acc) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " firmware_timestamp: " << m_gpu_metrics_tbl.m_firmware_timestamp << "\n"; + ss << " firmware_timestamp: " << m_gpu_metrics_tbl.m_firmware_timestamp << "\n"; - ostrstream << " current_gfxclk: " << "\n"; + ss << " current_gfxclk: " << "\n"; idx = 0; for (const auto& temp : m_gpu_metrics_tbl.m_current_gfxclk) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " current_socclk: " << "\n"; + ss << " current_socclk: " << "\n"; idx = 0; for (const auto& temp : m_gpu_metrics_tbl.m_current_socclk) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " current_vclk0: " << "\n"; + ss << " current_vclk0: " << "\n"; idx = 0; for (const auto& temp : m_gpu_metrics_tbl.m_current_vclk0) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " current_dclk0: " << "\n"; + ss << " current_dclk0: " << "\n"; idx = 0; for (const auto& temp : m_gpu_metrics_tbl.m_current_dclk0) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " padding: " << m_gpu_metrics_tbl.m_padding << "\n"; - LOG_DEBUG(ostrstream); + ss << " padding: " << m_gpu_metrics_tbl.m_padding << "\n"; + LOG_DEBUG(ss); } -rsmi_status_t GpuMetricsBase_v14_t::populate_metrics_dynamic_tbl() -{ - std::ostringstream ostrstream; +rsmi_status_t GpuMetricsBase_v14_t::populate_metrics_dynamic_tbl() { + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); + + if (!m_metrics_dynamic_tbl.empty()) { + m_metrics_dynamic_tbl.clear(); + } // // Note: Any metric treatment/changes (if any) should happen before they // get written to internal/external tables. // auto run_metric_adjustments_v14 = [&]() { - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; const auto gpu_metrics_version = translate_flag_to_metric_version(get_gpu_metrics_version_used()); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= info ======= " << " | Applying adjustments " << " | Metric Version: " << stringfy_metric_header_version( disjoin_metrics_version(gpu_metrics_version)) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); // firmware_timestamp is at 10ns resolution - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= Changes ======= " << " | {m_firmware_timestamp} from: " << m_gpu_metrics_tbl.m_firmware_timestamp << " to: " << (m_gpu_metrics_tbl.m_firmware_timestamp * 10); m_gpu_metrics_tbl.m_firmware_timestamp = (m_gpu_metrics_tbl.m_firmware_timestamp * 10); - LOG_DEBUG(ostrstream); + LOG_DEBUG(ss); }; @@ -1102,22 +1559,22 @@ rsmi_status_t GpuMetricsBase_v14_t::populate_metrics_dynamic_tbl() "current_uclk")) ); - ostrstream << __PRETTY_FUNCTION__ - << " | ======= end ======= " - << " | Success " - << " | Returning = " << getRSMIStatusString(status_code) - << " |"; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ + << " | ======= end ======= " + << " | Success " + << " | Returning = " << getRSMIStatusString(status_code) + << " |"; + LOG_TRACE(ss); return status_code; } rsmi_status_t init_max_public_gpu_matrics(AMGpuMetricsPublicLatest_t& rsmi_gpu_metrics) { - std::ostringstream ostrstream; + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); rsmi_gpu_metrics.temperature_edge = init_max_uint_types(); rsmi_gpu_metrics.temperature_hotspot = init_max_uint_types(); @@ -1206,24 +1663,250 @@ rsmi_status_t init_max_public_gpu_matrics(AMGpuMetricsPublicLatest_t& rsmi_gpu_m rsmi_gpu_metrics.pcie_nak_sent_count_acc = init_max_uint_types(); rsmi_gpu_metrics.pcie_nak_rcvd_count_acc = init_max_uint_types(); + rsmi_gpu_metrics.accumulation_counter = init_max_uint_types(); + rsmi_gpu_metrics.prochot_residency_acc = init_max_uint_types(); + rsmi_gpu_metrics.ppt_residency_acc = init_max_uint_types(); + rsmi_gpu_metrics.socket_thm_residency_acc = init_max_uint_types(); + rsmi_gpu_metrics.vr_thm_residency_acc = init_max_uint_types(); + rsmi_gpu_metrics.hbm_thm_residency_acc = init_max_uint_types(); + rsmi_gpu_metrics.num_partition = init_max_uint_types(); - ostrstream << __PRETTY_FUNCTION__ - << " | ======= end ======= " - << " | Success " - << " | Returning = " << getRSMIStatusString(status_code) - << " |"; - LOG_TRACE(ostrstream); + rsmi_gpu_metrics.pcie_lc_perf_other_end_recovery = + init_max_uint_types(); + + for (auto& row : rsmi_gpu_metrics.xcp_stats) { + std::fill(std::begin(row.gfx_busy_inst), std::end(row.gfx_busy_inst), + init_max_uint_types()); + std::fill(std::begin(row.jpeg_busy), std::end(row.jpeg_busy), + init_max_uint_types()); + std::fill(std::begin(row.vcn_busy), std::end(row.vcn_busy), + init_max_uint_types()); + std::fill(std::begin(row.gfx_busy_acc), std::end(row.gfx_busy_acc), + init_max_uint_types()); + } + + ss << __PRETTY_FUNCTION__ + << " | ======= end ======= " + << " | Success " + << " | Returning = " << getRSMIStatusString(status_code) + << " |"; + LOG_TRACE(ss); return status_code; } +AMGpuMetricsPublicLatestTupl_t GpuMetricsBase_v16_t::copy_internal_to_external_metrics() +{ + std::ostringstream ss; + auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); + + auto copy_data_from_internal_metrics_tbl = [&]() { + AMGpuMetricsPublicLatest_t metrics_public_init{}; + + // + // Note: Initializing data members with their max. If field is max, + // no data was assigned to it. + init_max_public_gpu_matrics(metrics_public_init); + + // Header + metrics_public_init.common_header.structure_size = m_gpu_metrics_tbl.m_common_header.m_structure_size; + metrics_public_init.common_header.format_revision = m_gpu_metrics_tbl.m_common_header.m_format_revision; + metrics_public_init.common_header.content_revision = m_gpu_metrics_tbl.m_common_header.m_content_revision; + + + // Temperature + metrics_public_init.temperature_hotspot = m_gpu_metrics_tbl.m_temperature_hotspot; + metrics_public_init.temperature_mem = m_gpu_metrics_tbl.m_temperature_mem; + metrics_public_init.temperature_vrsoc = m_gpu_metrics_tbl.m_temperature_vrsoc; + + // Power + metrics_public_init.current_socket_power = m_gpu_metrics_tbl.m_current_socket_power; + + // Utilization + metrics_public_init.average_gfx_activity = m_gpu_metrics_tbl.m_average_gfx_activity; + metrics_public_init.average_umc_activity = m_gpu_metrics_tbl.m_average_umc_activity; + + // Power/Energy + metrics_public_init.energy_accumulator = m_gpu_metrics_tbl.m_energy_accumulator; + + // Driver attached timestamp (in ns) + metrics_public_init.system_clock_counter = m_gpu_metrics_tbl.m_system_clock_counter; + + // Clock Lock Status. Each bit corresponds to clock instance + metrics_public_init.gfxclk_lock_status = m_gpu_metrics_tbl.m_gfxclk_lock_status; + + // Link width (number of lanes) and speed + metrics_public_init.pcie_link_width = m_gpu_metrics_tbl.m_pcie_link_width; + metrics_public_init.pcie_link_speed = m_gpu_metrics_tbl.m_pcie_link_speed; + + // XGMI bus width and bitrate + metrics_public_init.xgmi_link_width = m_gpu_metrics_tbl.m_xgmi_link_width; + metrics_public_init.xgmi_link_speed = m_gpu_metrics_tbl.m_xgmi_link_speed; + + // Utilization Accumulated + metrics_public_init.gfx_activity_acc = m_gpu_metrics_tbl.m_gfx_activity_acc; + metrics_public_init.mem_activity_acc = m_gpu_metrics_tbl.m_mem_activity_acc; + + // PCIE accumulated bandwidth + metrics_public_init.pcie_bandwidth_acc = m_gpu_metrics_tbl.m_pcie_bandwidth_acc; + + // PCIE instantaneous bandwidth + metrics_public_init.pcie_bandwidth_inst = m_gpu_metrics_tbl.m_pcie_bandwidth_inst; + + // PCIE L0 to recovery state transition accumulated count + metrics_public_init.pcie_l0_to_recov_count_acc = m_gpu_metrics_tbl.m_pcie_l0_to_recov_count_acc; + + // PCIE replay accumulated count + metrics_public_init.pcie_replay_count_acc = m_gpu_metrics_tbl.m_pcie_replay_count_acc; + + // PCIE replay rollover accumulated count + metrics_public_init.pcie_replay_rover_count_acc = m_gpu_metrics_tbl.m_pcie_replay_rover_count_acc; + + // PCIE NAK sent accumulated count + metrics_public_init.pcie_nak_sent_count_acc = m_gpu_metrics_tbl.m_pcie_nak_sent_count_acc; + + // PCIE NAK received accumulated count + metrics_public_init.pcie_nak_rcvd_count_acc = m_gpu_metrics_tbl.m_pcie_nak_rcvd_count_acc; + + // Accumulated throttler residencies + // bumped up public to uint64_t due to planned size increase for newer ASICs + metrics_public_init.accumulation_counter = m_gpu_metrics_tbl.m_accumulation_counter; + metrics_public_init.prochot_residency_acc = m_gpu_metrics_tbl.m_prochot_residency_acc; + metrics_public_init.ppt_residency_acc = m_gpu_metrics_tbl.m_ppt_residency_acc; + metrics_public_init.socket_thm_residency_acc = m_gpu_metrics_tbl.m_socket_thm_residency_acc; + metrics_public_init.vr_thm_residency_acc = m_gpu_metrics_tbl.m_vr_thm_residency_acc; + metrics_public_init.hbm_thm_residency_acc = m_gpu_metrics_tbl.m_hbm_thm_residency_acc; + + // XGMI accumulated data transfer size + // xgmi_read_data + const auto xgmi_read_data_num_elems = + static_cast( + std::end(m_gpu_metrics_tbl.m_xgmi_read_data_acc) - + std::begin(m_gpu_metrics_tbl.m_xgmi_read_data_acc)); + std::copy_n(std::begin(m_gpu_metrics_tbl.m_xgmi_read_data_acc), + xgmi_read_data_num_elems, + metrics_public_init.xgmi_read_data_acc); + // xgmi_write_data + const auto xgmi_write_data_num_elems = + static_cast( + std::end(m_gpu_metrics_tbl.m_xgmi_write_data_acc) - + std::begin(m_gpu_metrics_tbl.m_xgmi_write_data_acc)); + std::copy_n(std::begin(m_gpu_metrics_tbl.m_xgmi_write_data_acc), + xgmi_write_data_num_elems, + metrics_public_init.xgmi_write_data_acc); + + // PMFW attached timestamp (10ns resolution) + metrics_public_init.firmware_timestamp = m_gpu_metrics_tbl.m_firmware_timestamp; + + // Current clocks + // current_gfxclk + const auto curr_gfxclk_num_elems = + static_cast( + std::end(m_gpu_metrics_tbl.m_current_gfxclk) - + std::begin(m_gpu_metrics_tbl.m_current_gfxclk)); + std::copy_n(std::begin(m_gpu_metrics_tbl.m_current_gfxclk), + curr_gfxclk_num_elems, + metrics_public_init.current_gfxclks); + + // current_socclk + const auto curr_socclk_num_elems = + static_cast( + std::end(m_gpu_metrics_tbl.m_current_socclk) - + std::begin(m_gpu_metrics_tbl.m_current_socclk)); + std::copy_n(std::begin(m_gpu_metrics_tbl.m_current_socclk), + curr_socclk_num_elems, + metrics_public_init.current_socclks); + + // current_vclk0 + const auto curr_vclk0_num_elems = + static_cast( + std::end(m_gpu_metrics_tbl.m_current_vclk0) - + std::begin(m_gpu_metrics_tbl.m_current_vclk0)); + std::copy_n(std::begin(m_gpu_metrics_tbl.m_current_vclk0), + curr_vclk0_num_elems, + metrics_public_init.current_vclk0s); + + // current_dclk0 + const auto curr_dclk0_num_elems = + static_cast( + std::end(m_gpu_metrics_tbl.m_current_dclk0) - + std::begin(m_gpu_metrics_tbl.m_current_dclk0)); + std::copy_n(std::begin(m_gpu_metrics_tbl.m_current_dclk0), + curr_dclk0_num_elems, + metrics_public_init.current_dclk0s); + + metrics_public_init.current_uclk = m_gpu_metrics_tbl.m_current_uclk; + + metrics_public_init.num_partition = m_gpu_metrics_tbl.m_num_partition; + + metrics_public_init.pcie_lc_perf_other_end_recovery = + m_gpu_metrics_tbl.m_pcie_lc_perf_other_end_recovery; + + auto priv_it = std::begin(m_gpu_metrics_tbl.m_xcp_stats); + for (auto pub_it = std::begin(metrics_public_init.xcp_stats); + pub_it != std::end(metrics_public_init.xcp_stats); + ++pub_it, ++priv_it) { + std::copy_n(std::begin(priv_it->gfx_busy_inst), RSMI_MAX_NUM_XCC, + pub_it->gfx_busy_inst); + std::copy_n(std::begin(priv_it->jpeg_busy), RSMI_MAX_NUM_JPEG_ENGS, + pub_it->jpeg_busy); + std::copy_n(std::begin(priv_it->vcn_busy), RSMI_MAX_NUM_VCNS, + pub_it->vcn_busy); + std::copy_n(std::begin(priv_it->gfx_busy_acc), RSMI_MAX_NUM_XCC, + pub_it->gfx_busy_acc); + } + + // + // Note: Backwards compatibility -> Handling extra/exception cases + // related to earlier versions (1.3/1.4/1.5) + metrics_public_init.current_gfxclk = metrics_public_init.current_gfxclks[0]; + + metrics_public_init.current_socclk = metrics_public_init.current_socclks[0]; + + metrics_public_init.current_vclk0 = metrics_public_init.current_vclk0s[0]; + + metrics_public_init.current_vclk1 = metrics_public_init.current_vclk0s[1]; + + metrics_public_init.current_dclk0 = metrics_public_init.current_dclk0s[0]; + + metrics_public_init.current_dclk1 = metrics_public_init.current_dclk0s[1]; + + // separate by XCP + if (this->m_partition_id < kRSMI_MAX_NUM_XCP + && m_gpu_metrics_tbl.m_xcp_stats[this->m_partition_id].vcn_busy[0] != UINT16_MAX) { + std::copy(std::begin(m_gpu_metrics_tbl.m_xcp_stats[this->m_partition_id].vcn_busy), + std::end(m_gpu_metrics_tbl.m_xcp_stats[this->m_partition_id].vcn_busy), + std::begin(metrics_public_init.vcn_activity)); + } + if (this->m_partition_id < kRSMI_MAX_NUM_XCP + && m_gpu_metrics_tbl.m_xcp_stats[this->m_partition_id].jpeg_busy[0] != UINT16_MAX) { + std::copy(std::begin(m_gpu_metrics_tbl.m_xcp_stats[this->m_partition_id].jpeg_busy), + std::end(m_gpu_metrics_tbl.m_xcp_stats[this->m_partition_id].jpeg_busy), + std::begin(metrics_public_init.jpeg_activity)); + } + + return metrics_public_init; + }(); + + ss << __PRETTY_FUNCTION__ + << " | ======= end ======= " + << " | Success " + << " | Returning = " << getRSMIStatusString(status_code) + << " |"; + LOG_TRACE(ss); + + return std::make_tuple(status_code, copy_data_from_internal_metrics_tbl); +} + AMGpuMetricsPublicLatestTupl_t GpuMetricsBase_v15_t::copy_internal_to_external_metrics() { - std::ostringstream ostrstream; + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); auto copy_data_from_internal_metrics_tbl = [&]() { AMGpuMetricsPublicLatest_t metrics_public_init{}; @@ -1378,42 +2061,42 @@ AMGpuMetricsPublicLatestTupl_t GpuMetricsBase_v15_t::copy_internal_to_external_m // Note: Backwards compatibility -> Handling extra/exception cases // related to earlier versions (1.3) metrics_public_init.current_gfxclk = metrics_public_init.current_gfxclks[0]; - metrics_public_init.average_gfxclk_frequency = metrics_public_init.current_gfxclks[0]; + // metrics_public_init.average_gfxclk_frequency = metrics_public_init.current_gfxclks[0]; metrics_public_init.current_socclk = metrics_public_init.current_socclks[0]; - metrics_public_init.average_socclk_frequency = metrics_public_init.current_socclks[0]; + // metrics_public_init.average_socclk_frequency = metrics_public_init.current_socclks[0]; metrics_public_init.current_vclk0 = metrics_public_init.current_vclk0s[0]; - metrics_public_init.average_vclk0_frequency = metrics_public_init.current_vclk0s[0]; + // metrics_public_init.average_vclk0_frequency = metrics_public_init.current_vclk0s[0]; metrics_public_init.current_vclk1 = metrics_public_init.current_vclk0s[1]; - metrics_public_init.average_vclk1_frequency = metrics_public_init.current_vclk0s[1]; + // metrics_public_init.average_vclk1_frequency = metrics_public_init.current_vclk0s[1]; metrics_public_init.current_dclk0 = metrics_public_init.current_dclk0s[0]; - metrics_public_init.average_dclk0_frequency = metrics_public_init.current_dclk0s[0]; + // metrics_public_init.average_dclk0_frequency = metrics_public_init.current_dclk0s[0]; metrics_public_init.current_dclk1 = metrics_public_init.current_dclk0s[1]; - metrics_public_init.average_dclk1_frequency = metrics_public_init.current_dclk0s[1]; + // metrics_public_init.average_dclk1_frequency = metrics_public_init.current_dclk0s[1]; return metrics_public_init; }(); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return std::make_tuple(status_code, copy_data_from_internal_metrics_tbl); } AMGpuMetricsPublicLatestTupl_t GpuMetricsBase_v14_t::copy_internal_to_external_metrics() { - std::ostringstream ostrstream; + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); auto copy_data_from_internal_metrics_tbl = [&]() { AMGpuMetricsPublicLatest_t metrics_public_init{}; @@ -1553,47 +2236,47 @@ AMGpuMetricsPublicLatestTupl_t GpuMetricsBase_v14_t::copy_internal_to_external_m // Note: Backwards compatibility -> Handling extra/exception cases // related to earlier versions (1.3) metrics_public_init.current_gfxclk = metrics_public_init.current_gfxclks[0]; - metrics_public_init.average_gfxclk_frequency = metrics_public_init.current_gfxclks[0]; + // metrics_public_init.average_gfxclk_frequency = metrics_public_init.current_gfxclks[0]; metrics_public_init.current_socclk = metrics_public_init.current_socclks[0]; - metrics_public_init.average_socclk_frequency = metrics_public_init.current_socclks[0]; + // metrics_public_init.average_socclk_frequency = metrics_public_init.current_socclks[0]; metrics_public_init.current_vclk0 = metrics_public_init.current_vclk0s[0]; - metrics_public_init.average_vclk0_frequency = metrics_public_init.current_vclk0s[0]; + // metrics_public_init.average_vclk0_frequency = metrics_public_init.current_vclk0s[0]; metrics_public_init.current_vclk1 = metrics_public_init.current_vclk0s[1]; - metrics_public_init.average_vclk1_frequency = metrics_public_init.current_vclk0s[1]; + // metrics_public_init.average_vclk1_frequency = metrics_public_init.current_vclk0s[1]; metrics_public_init.current_dclk0 = metrics_public_init.current_dclk0s[0]; - metrics_public_init.average_dclk0_frequency = metrics_public_init.current_dclk0s[0]; + // metrics_public_init.average_dclk0_frequency = metrics_public_init.current_dclk0s[0]; metrics_public_init.current_dclk1 = metrics_public_init.current_dclk0s[1]; - metrics_public_init.average_dclk1_frequency = metrics_public_init.current_dclk0s[1]; + // metrics_public_init.average_dclk1_frequency = metrics_public_init.current_dclk0s[1]; return metrics_public_init; }(); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return std::make_tuple(status_code, copy_data_from_internal_metrics_tbl); } void GpuMetricsBase_v13_t::dump_internal_metrics_table() { - std::ostringstream ostrstream; + std::ostringstream ss; std::cout << __PRETTY_FUNCTION__ << " | ======= start ======= \n"; - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= DEBUG ======= " << " | Metric Version: " << stringfy_metric_header_version(m_gpu_metrics_tbl.m_common_header) << " | Size: " << print_unsigned_int(m_gpu_metrics_tbl.m_common_header.m_structure_size) << " |" << "\n"; - ostrstream << " temperature_edge: " << m_gpu_metrics_tbl.m_temperature_edge << "\n" + ss << " temperature_edge: " << m_gpu_metrics_tbl.m_temperature_edge << "\n" << " temperature_hotspot: " << m_gpu_metrics_tbl.m_temperature_hotspot << "\n" << " temperature_mem: " << m_gpu_metrics_tbl.m_temperature_mem << "\n" << " temperature_vrgfx: " << m_gpu_metrics_tbl.m_temperature_vrgfx << "\n" @@ -1635,16 +2318,16 @@ void GpuMetricsBase_v13_t::dump_internal_metrics_table() << " gfx_activity_acc: " << m_gpu_metrics_tbl.m_gfx_activity_acc << "\n" << " mem_activity_acc: " << m_gpu_metrics_tbl.m_mem_activity_acc << "\n"; - LOG_DEBUG(ostrstream); + LOG_DEBUG(ss); - ostrstream << " temperature_hbm: " << "\n"; + ss << " temperature_hbm: " << "\n"; auto idx = uint64_t(0); for (const auto& temp : m_gpu_metrics_tbl.m_temperature_hbm) { - ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ss << "\t [" << idx << "]: " << temp << "\n"; ++idx; } - ostrstream << " firmware_timestamp: " << m_gpu_metrics_tbl.m_firmware_timestamp << "\n" + ss << " firmware_timestamp: " << m_gpu_metrics_tbl.m_firmware_timestamp << "\n" << " voltage_soc: " << m_gpu_metrics_tbl.m_voltage_soc << "\n" << " voltage_gfx: " << m_gpu_metrics_tbl.m_voltage_gfx << "\n" @@ -1652,38 +2335,41 @@ void GpuMetricsBase_v13_t::dump_internal_metrics_table() << " padding1: " << m_gpu_metrics_tbl.m_padding1 << "\n" << " m_indep_throttle_status: " << m_gpu_metrics_tbl.m_indep_throttle_status << "\n"; - LOG_DEBUG(ostrstream); + LOG_DEBUG(ss); } -rsmi_status_t GpuMetricsBase_v13_t::populate_metrics_dynamic_tbl() -{ - std::ostringstream ostrstream; +rsmi_status_t GpuMetricsBase_v13_t::populate_metrics_dynamic_tbl() { + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); + + if (!m_metrics_dynamic_tbl.empty()) { + m_metrics_dynamic_tbl.clear(); + } // // Note: Any metric treatment/changes (if any) should happen before they // get written to internal/external tables. // auto run_metric_adjustments_v13 = [&]() { - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; const auto gpu_metrics_version = translate_flag_to_metric_version(get_gpu_metrics_version_used()); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= info ======= " << " | Applying adjustments " << " | Metric Version: " << stringfy_metric_header_version( disjoin_metrics_version(gpu_metrics_version)) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); // firmware_timestamp is at 10ns resolution - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= Changes ======= " << " | {m_firmware_timestamp} from: " << m_gpu_metrics_tbl.m_firmware_timestamp << " to: " << (m_gpu_metrics_tbl.m_firmware_timestamp * 10); m_gpu_metrics_tbl.m_firmware_timestamp = (m_gpu_metrics_tbl.m_firmware_timestamp * 10); - LOG_DEBUG(ostrstream); + LOG_DEBUG(ss); }; @@ -1900,22 +2586,22 @@ rsmi_status_t GpuMetricsBase_v13_t::populate_metrics_dynamic_tbl() "voltage_mem")) ); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return status_code; } AMGpuMetricsPublicLatestTupl_t GpuMetricsBase_v13_t::copy_internal_to_external_metrics() { - std::ostringstream ostrstream; + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); auto copy_data_from_internal_metrics_tbl = [&]() { AMGpuMetricsPublicLatest_t metrics_public_init{}; @@ -1961,9 +2647,13 @@ AMGpuMetricsPublicLatestTupl_t GpuMetricsBase_v13_t::copy_internal_to_external_m // Current clocks metrics_public_init.current_gfxclk = m_gpu_metrics_tbl.m_current_gfxclk; + metrics_public_init.current_gfxclks[0] = m_gpu_metrics_tbl.m_current_gfxclk; metrics_public_init.current_socclk = m_gpu_metrics_tbl.m_current_socclk; + metrics_public_init.current_socclks[0] = m_gpu_metrics_tbl.m_current_socclk; metrics_public_init.current_vclk0 = m_gpu_metrics_tbl.m_current_vclk0; + metrics_public_init.current_vclk0s[0] = m_gpu_metrics_tbl.m_current_vclk0; metrics_public_init.current_dclk0 = m_gpu_metrics_tbl.m_current_dclk0; + metrics_public_init.current_dclk0s[0] = m_gpu_metrics_tbl.m_current_dclk0; metrics_public_init.current_uclk = m_gpu_metrics_tbl.m_current_uclk; metrics_public_init.current_vclk1 = m_gpu_metrics_tbl.m_current_vclk1; metrics_public_init.current_dclk1 = m_gpu_metrics_tbl.m_current_dclk1; @@ -2004,50 +2694,56 @@ AMGpuMetricsPublicLatestTupl_t GpuMetricsBase_v13_t::copy_internal_to_external_m // // Note: Backwards compatibility -> Handling extra/exception cases // related to earlier versions (1.2) - metrics_public_init.current_socket_power = metrics_public_init.average_socket_power; + // metrics_public_init.current_socket_power = metrics_public_init.average_socket_power; + if (metrics_public_init.average_mm_activity != UINT16_MAX) { + metrics_public_init.vcn_activity[0] = metrics_public_init.average_mm_activity; + } return metrics_public_init; }(); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return std::make_tuple(status_code, copy_data_from_internal_metrics_tbl); } -rsmi_status_t GpuMetricsBase_v12_t::populate_metrics_dynamic_tbl() -{ - std::ostringstream ostrstream; +rsmi_status_t GpuMetricsBase_v12_t::populate_metrics_dynamic_tbl() { + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); + + if (!m_metrics_dynamic_tbl.empty()) { + m_metrics_dynamic_tbl.clear(); + } // // Note: Any metric treatment/changes (if any) should happen before they // get written to internal/external tables. // auto run_metric_adjustments_v12 = [&]() { - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; const auto gpu_metrics_version = translate_flag_to_metric_version(get_gpu_metrics_version_used()); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= info ======= " << " | Applying adjustments " << " | Metric Version: " << stringfy_metric_header_version( disjoin_metrics_version(gpu_metrics_version)) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); // firmware_timestamp is at 10ns resolution - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= Changes ======= " << " | {m_firmware_timestamp} from: " << m_gpu_metrics_tbl.m_firmware_timestamp << " to: " << (m_gpu_metrics_tbl.m_firmware_timestamp * 10); m_gpu_metrics_tbl.m_firmware_timestamp = (m_gpu_metrics_tbl.m_firmware_timestamp * 10); - LOG_DEBUG(ostrstream); + LOG_DEBUG(ss); }; @@ -2242,22 +2938,22 @@ rsmi_status_t GpuMetricsBase_v12_t::populate_metrics_dynamic_tbl() "pcie_link_speed")) ); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return status_code; } AMGpuMetricsPublicLatestTupl_t GpuMetricsBase_v12_t::copy_internal_to_external_metrics() { - std::ostringstream ostrstream; + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); auto copy_data_from_internal_metrics_tbl = [&]() { AMGpuMetricsPublicLatest_t metrics_public_init{}; @@ -2343,37 +3039,40 @@ AMGpuMetricsPublicLatestTupl_t GpuMetricsBase_v12_t::copy_internal_to_external_m return metrics_public_init; }(); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return std::make_tuple(status_code, copy_data_from_internal_metrics_tbl); } -rsmi_status_t GpuMetricsBase_v11_t::populate_metrics_dynamic_tbl() -{ - std::ostringstream ostrstream; +rsmi_status_t GpuMetricsBase_v11_t::populate_metrics_dynamic_tbl() { + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); + + if (!m_metrics_dynamic_tbl.empty()) { + m_metrics_dynamic_tbl.clear(); + } // // Note: Any metric treatment/changes (if any) should happen before they // get written to internal/external tables. // auto run_metric_adjustments_v11 = [&]() { - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; const auto gpu_metrics_version = translate_flag_to_metric_version(get_gpu_metrics_version_used()); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= info ======= " << " | Applying adjustments " << " | Metric Version: " << stringfy_metric_header_version( disjoin_metrics_version(gpu_metrics_version)) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); }; @@ -2563,22 +3262,22 @@ rsmi_status_t GpuMetricsBase_v11_t::populate_metrics_dynamic_tbl() "pcie_link_speed")) ); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return status_code; } AMGpuMetricsPublicLatestTupl_t GpuMetricsBase_v11_t::copy_internal_to_external_metrics() { - std::ostringstream ostrstream; + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); auto copy_data_from_internal_metrics_tbl = [&]() { AMGpuMetricsPublicLatest_t metrics_public_init{}; @@ -2661,12 +3360,12 @@ AMGpuMetricsPublicLatestTupl_t GpuMetricsBase_v11_t::copy_internal_to_external_m return metrics_public_init; }(); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return std::make_tuple(status_code, copy_data_from_internal_metrics_tbl); } @@ -2674,10 +3373,10 @@ AMGpuMetricsPublicLatestTupl_t GpuMetricsBase_v11_t::copy_internal_to_external_m rsmi_status_t Device::dev_read_gpu_metrics_header_data() { - std::ostringstream ostrstream; + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); // Check if/when metrics table needs to be refreshed. auto op_result = readDevInfo(DevInfoTypes::kDevGpuMetrics, @@ -2685,7 +3384,7 @@ rsmi_status_t Device::dev_read_gpu_metrics_header_data() &m_gpu_metrics_header); if ((status_code = ErrnoToRsmiStatus(op_result)) != rsmi_status_t::RSMI_STATUS_SUCCESS) { - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Device #: " << index() @@ -2696,12 +3395,12 @@ rsmi_status_t Device::dev_read_gpu_metrics_header_data() << " Could not read Metrics Header: " << print_unsigned_int(m_gpu_metrics_header.m_structure_size) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return status_code; } if ((status_code = is_gpu_metrics_version_supported(m_gpu_metrics_header)) == rsmi_status_t::RSMI_STATUS_NOT_SUPPORTED) { - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Device #: " << index() @@ -2712,12 +3411,12 @@ rsmi_status_t Device::dev_read_gpu_metrics_header_data() << " Could not read Metrics Header: " << print_unsigned_int(m_gpu_metrics_header.m_structure_size) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return status_code; } m_gpu_metrics_updated_timestamp = actual_timestamp_in_secs(); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Device #: " << index() @@ -2726,16 +3425,16 @@ rsmi_status_t Device::dev_read_gpu_metrics_header_data() << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return status_code; } rsmi_status_t Device::dev_read_gpu_metrics_all_data() { - std::ostringstream ostrstream; + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); // At this point we should have a valid gpu_metrics pointer, and // we already read the header; setup_gpu_metrics_reading() @@ -2744,7 +3443,7 @@ rsmi_status_t Device::dev_read_gpu_metrics_all_data() (!m_gpu_metrics_header.m_format_revision) || (!m_gpu_metrics_header.m_content_revision))) { status_code = rsmi_status_t::RSMI_STATUS_SETTING_UNAVAILABLE; - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Device #: " << index() @@ -2753,7 +3452,7 @@ rsmi_status_t Device::dev_read_gpu_metrics_all_data() << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return status_code; } @@ -2762,7 +3461,7 @@ rsmi_status_t Device::dev_read_gpu_metrics_all_data() m_gpu_metrics_ptr->get_metrics_table().get()); if ((status_code = ErrnoToRsmiStatus(op_result)) != rsmi_status_t::RSMI_STATUS_SUCCESS) { - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Device #: " << index() @@ -2773,14 +3472,14 @@ rsmi_status_t Device::dev_read_gpu_metrics_all_data() << " Could not read Metrics Header: " << print_unsigned_int(m_gpu_metrics_header.m_structure_size) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return status_code; } // All metric units are pushed in. status_code = m_gpu_metrics_ptr->populate_metrics_dynamic_tbl(); if (status_code != rsmi_status_t::RSMI_STATUS_SUCCESS) { - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Device #: " << index() @@ -2789,11 +3488,11 @@ rsmi_status_t Device::dev_read_gpu_metrics_all_data() << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); } m_gpu_metrics_updated_timestamp = actual_timestamp_in_secs(); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Device #: " << index() @@ -2802,16 +3501,16 @@ rsmi_status_t Device::dev_read_gpu_metrics_all_data() << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return status_code; } rsmi_status_t Device::setup_gpu_metrics_reading() { - std::ostringstream ostrstream; + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); status_code = dev_read_gpu_metrics_header_data(); if (status_code != rsmi_status_t::RSMI_STATUS_SUCCESS) { @@ -2821,7 +3520,7 @@ rsmi_status_t Device::setup_gpu_metrics_reading() const auto gpu_metrics_flag_version = translate_header_to_flag_version(dev_get_metrics_header()); if (gpu_metrics_flag_version == AMDGpuMetricVersionFlags_t::kGpuMetricNone) { status_code = rsmi_status_t::RSMI_STATUS_NOT_SUPPORTED; - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Device #: " << index() @@ -2832,7 +3531,7 @@ rsmi_status_t Device::setup_gpu_metrics_reading() << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return status_code; } @@ -2841,7 +3540,7 @@ rsmi_status_t Device::setup_gpu_metrics_reading() m_gpu_metrics_ptr = amdgpu_metrics_factory(gpu_metrics_flag_version); if (!m_gpu_metrics_ptr) { status_code = rsmi_status_t::RSMI_STATUS_UNEXPECTED_DATA; - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Device #: " << index() @@ -2850,15 +3549,17 @@ rsmi_status_t Device::setup_gpu_metrics_reading() << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return status_code; } + m_gpu_metrics_ptr->set_device_id(m_device_id); + m_gpu_metrics_ptr->set_partition_id(m_partition_id); // // m_gpu_metrics_ptr has the pointer to the proper object type/version. status_code = dev_read_gpu_metrics_all_data(); if (status_code != rsmi_status_t::RSMI_STATUS_SUCCESS) { - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Device #: " << index() @@ -2867,11 +3568,11 @@ rsmi_status_t Device::setup_gpu_metrics_reading() << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return status_code; } - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Device #: " << index() @@ -2881,7 +3582,7 @@ rsmi_status_t Device::setup_gpu_metrics_reading() << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return status_code; } @@ -2919,13 +3620,12 @@ auto get_casted_value(const AMDGpuDynamicMetricsValue_t& metrics_value) } -rsmi_status_t Device::dev_log_gpu_metrics(std::ostringstream& outstream_metrics) -{ - std::ostringstream ostrstream; +rsmi_status_t Device::dev_log_gpu_metrics(std::ostringstream& outstream_metrics) { + std::ostringstream ss; std::ostringstream tmp_outstream_metrics; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); // If we still don't have a valid gpu_metrics pointer; // meaning, we didn't run any queries, and just want to @@ -2935,7 +3635,7 @@ rsmi_status_t Device::dev_log_gpu_metrics(std::ostringstream& outstream_metrics) if ((status_code != rsmi_status_t::RSMI_STATUS_SUCCESS) || (!m_gpu_metrics_ptr)) { // At this point we should have a valid gpu_metrics pointer. status_code = rsmi_status_t::RSMI_STATUS_UNEXPECTED_DATA; - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Device #: " << index() @@ -2944,7 +3644,7 @@ rsmi_status_t Device::dev_log_gpu_metrics(std::ostringstream& outstream_metrics) << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return status_code; } @@ -3030,7 +3730,7 @@ rsmi_status_t Device::dev_log_gpu_metrics(std::ostringstream& outstream_metrics) outstream_metrics << tmp_outstream_metrics.rdbuf(); LOG_DEBUG(tmp_outstream_metrics); - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Device #: " << index() @@ -3040,22 +3740,21 @@ rsmi_status_t Device::dev_log_gpu_metrics(std::ostringstream& outstream_metrics) << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return status_code; } AMGpuMetricsPublicLatestTupl_t Device::dev_copy_internal_to_external_metrics() { - std::ostringstream ostrstream; - std::ostringstream tmp_outstream_metrics; + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); if (!m_gpu_metrics_ptr) { // At this point we should have a valid gpu_metrics pointer. status_code = rsmi_status_t::RSMI_STATUS_UNEXPECTED_DATA; - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Device #: " << index() @@ -3064,11 +3763,11 @@ AMGpuMetricsPublicLatestTupl_t Device::dev_copy_internal_to_external_metrics() << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return std::make_tuple(status_code, AMGpuMetricsPublicLatest_t()); } - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Device #: " << index() @@ -3078,7 +3777,7 @@ AMGpuMetricsPublicLatestTupl_t Device::dev_copy_internal_to_external_metrics() << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return m_gpu_metrics_ptr->copy_internal_to_external_metrics(); } @@ -3086,15 +3785,15 @@ AMGpuMetricsPublicLatestTupl_t Device::dev_copy_internal_to_external_metrics() rsmi_status_t Device::run_internal_gpu_metrics_query(AMDGpuMetricsUnitType_t metric_counter, AMDGpuDynamicMetricTblValues_t& values) { - std::ostringstream ostrstream; + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_NOT_SUPPORTED); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); status_code = setup_gpu_metrics_reading(); if ((status_code != rsmi_status_t::RSMI_STATUS_SUCCESS) || (!m_gpu_metrics_ptr)) { status_code = rsmi_status_t::RSMI_STATUS_UNEXPECTED_DATA; - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Device #: " << index() @@ -3103,25 +3802,25 @@ rsmi_status_t Device::run_internal_gpu_metrics_query(AMDGpuMetricsUnitType_t met << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return status_code; } // Lookup the dynamic table - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= info ======= " << " | Device #: " << index() << " | Metric Version: " << stringfy_metrics_header(dev_get_metrics_header()) << " | Metric Unit: " << static_cast(metric_counter) << " |"; - LOG_INFO(ostrstream); + LOG_INFO(ss); const auto gpu_metrics_tbl = m_gpu_metrics_ptr->get_metrics_dynamic_tbl(); for (const auto& [metric_class, metric_data] : gpu_metrics_tbl) { for (const auto& [metric_unit, metric_values] : metric_data) { if (metric_unit == metric_counter) { values = metric_values; status_code = rsmi_status_t::RSMI_STATUS_SUCCESS; - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Device #: " << index() @@ -3130,13 +3829,13 @@ rsmi_status_t Device::run_internal_gpu_metrics_query(AMDGpuMetricsUnitType_t met << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return status_code; } } } - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Device #: " << index() @@ -3144,7 +3843,7 @@ rsmi_status_t Device::run_internal_gpu_metrics_query(AMDGpuMetricsUnitType_t met << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return status_code; } @@ -3177,10 +3876,10 @@ constexpr bool is_std_vector_type_supported_v() template rsmi_status_t rsmi_dev_gpu_metrics_info_query(uint32_t dv_ind, AMDGpuMetricsUnitType_t metric_counter, T& metric_value) { - std::ostringstream ostrstream; + std::ostringstream ss; auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - ostrstream << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ << " | ======= start ======="; + LOG_TRACE(ss); static constexpr bool is_supported_vector_type = [&]() { if constexpr (is_std_vector_v) { @@ -3198,7 +3897,7 @@ rsmi_status_t rsmi_dev_gpu_metrics_info_query(uint32_t dv_ind, AMDGpuMetricsUnit GET_DEV_FROM_INDX status_code = dev->run_internal_gpu_metrics_query(metric_counter, tmp_values); if ((status_code != rsmi_status_t::RSMI_STATUS_SUCCESS) || tmp_values.empty()) { - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Device #: " << dv_ind @@ -3210,7 +3909,7 @@ rsmi_status_t rsmi_dev_gpu_metrics_info_query(uint32_t dv_ind, AMDGpuMetricsUnit << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return status_code; } @@ -3233,14 +3932,14 @@ rsmi_status_t rsmi_dev_gpu_metrics_info_query(uint32_t dv_ind, AMDGpuMetricsUnit static_assert(is_dependent_false_v, "Error: Data Type not supported..."); } - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Device #: " << dv_ind << " | Metric Type: " << static_cast(metric_counter) << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return status_code; } @@ -3276,9 +3975,9 @@ rsmi_dev_gpu_metrics_header_info_get(uint32_t dv_ind, metrics_table_header_t& he { TRY auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); - std::ostringstream ostrstream; - ostrstream << __PRETTY_FUNCTION__ << "| ======= start ======="; - LOG_TRACE(ostrstream); + std::ostringstream ss; + ss << __PRETTY_FUNCTION__ << "| ======= start ======="; + LOG_TRACE(ss); GET_DEV_FROM_INDX status_code = dev->dev_read_gpu_metrics_header_data(); @@ -3287,14 +3986,14 @@ rsmi_dev_gpu_metrics_header_info_get(uint32_t dv_ind, metrics_table_header_t& he std::memcpy(&header_value, &tmp_header_info, sizeof(metrics_table_header_t)); } - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " << " | Device #: " << dv_ind << " | Returning = " << getRSMIStatusString(status_code) << " |"; - LOG_TRACE(ostrstream); + LOG_TRACE(ss); return status_code; CATCH @@ -3315,38 +4014,52 @@ rsmi_dev_gpu_metrics_info_get(uint32_t dv_ind, rsmi_gpu_metrics_t* smu) { auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); std::ostringstream ostrstream; - ostrstream << __PRETTY_FUNCTION__ << "| ======= start ======="; - LOG_TRACE(ostrstream); + std::ostringstream ss; + + ss << __PRETTY_FUNCTION__ << "| ======= start ======="; + LOG_TRACE(ss); assert(smu != nullptr); if (smu == nullptr) { status_code = rsmi_status_t::RSMI_STATUS_INVALID_ARGS; + ss << __PRETTY_FUNCTION__ + << " | ======= end ======= " + << " | Fail " + << " | Device #: " << dv_ind + << " | Returning = " + << getRSMIStatusString(status_code) + << " |"; + LOG_ERROR(ss); return status_code; } + dev->set_smi_device_id(dv_ind); + uint32_t partition_id = 0; + auto ret = rsmi_dev_partition_id_get(dv_ind, &partition_id); + dev->set_smi_partition_id(partition_id); dev->dev_log_gpu_metrics(ostrstream); const auto [error_code, external_metrics] = dev->dev_copy_internal_to_external_metrics(); if (error_code != rsmi_status_t::RSMI_STATUS_SUCCESS) { - ostrstream << __PRETTY_FUNCTION__ + ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Fail " << " | Device #: " << dv_ind << " | Returning = " << getRSMIStatusString(error_code) << " |"; - LOG_ERROR(ostrstream); + LOG_ERROR(ss); return error_code; } *smu = external_metrics; - ostrstream << __PRETTY_FUNCTION__ - << " | ======= end ======= " - << " | Success " - << " | Device #: " << dv_ind - << " | Returning = " - << getRSMIStatusString(status_code) - << " |"; - LOG_TRACE(ostrstream); + ss << __PRETTY_FUNCTION__ + << " | ======= end ======= " + << " | Success " + << " | Device #: " << dv_ind + << " | Returning = " + << getRSMIStatusString(status_code) + << " |"; + LOG_TRACE(ss); return status_code; CATCH diff --git a/projects/rocm-smi-lib/tests/rocm_smi_test/functional/gpu_metrics_read.cc b/projects/rocm-smi-lib/tests/rocm_smi_test/functional/gpu_metrics_read.cc index e7c5306ad0..2ec93cbdf6 100644 --- a/projects/rocm-smi-lib/tests/rocm_smi_test/functional/gpu_metrics_read.cc +++ b/projects/rocm-smi-lib/tests/rocm_smi_test/functional/gpu_metrics_read.cc @@ -5,7 +5,7 @@ * The University of Illinois/NCSA * Open Source License (NCSA) * - * Copyright (c) 2020, Advanced Micro Devices, Inc. + * Copyright (c) 2020-2024, Advanced Micro Devices, Inc. * All rights reserved. * * Developed by: @@ -46,7 +46,9 @@ #include #include +#include #include +#include #include #include #include @@ -89,36 +91,6 @@ void TestGpuMetricsRead::Close() { } -using GPUMetricResults_t = std::map; -GPUMetricResults_t MetricResults{}; - -template -auto print_error_or_value(std::string title, std::string func_name, const T& metric) { - auto str_values = title; - const auto status_code = MetricResults.at(func_name); - if (status_code == rsmi_status_t::RSMI_STATUS_SUCCESS) { - if constexpr (std::is_array_v) { - auto idx = uint16_t(0); - - const auto num_elems = static_cast(std::end(metric) - std::begin(metric)); - str_values += ("\n\t\t num of values: " + std::to_string(num_elems) + "\n"); - for (const auto& el : metric) { - str_values += "\t\t [" + std::to_string(idx) + "]: " + std::to_string(el) + "\n"; - ++idx; - } - return str_values; - } - else if constexpr ((std::is_same_v) || - (std::is_same_v) || - (std::is_same_v)) { - - return str_values += std::to_string(metric); - } - } - else { - return str_values += ("\n\t\tStatus: [" + std::to_string(status_code) + "] " + "-> " + amd::smi::getRSMIStatusString(status_code)); - } -}; void TestGpuMetricsRead::Run(void) { rsmi_status_t err; @@ -140,13 +112,15 @@ void TestGpuMetricsRead::Run(void) { auto ret = rsmi_dev_metrics_header_info_get(i, &header_values); if (ret == rsmi_status_t::RSMI_STATUS_SUCCESS) { std::cout << "\t[Metrics Header]" << "\n"; - std::cout << "\t -> format_revision : " << amd::smi::print_unsigned_int(header_values.format_revision) << "\n"; - std::cout << "\t -> content_revision : " << amd::smi::print_unsigned_int(header_values.content_revision) << "\n"; + std::cout << "\t -> format_revision : " + << static_cast(header_values.format_revision) << "\n"; + std::cout << "\t -> content_revision : " + << static_cast(header_values.content_revision) << "\n"; std::cout << "\t--------------------" << "\n"; } } - rsmi_gpu_metrics_t smu; + rsmi_gpu_metrics_t smu = {}; err = rsmi_dev_gpu_metrics_info_get(i, &smu); if (err != RSMI_STATUS_SUCCESS) { if (err == RSMI_STATUS_NOT_SUPPORTED) { @@ -159,96 +133,232 @@ void TestGpuMetricsRead::Run(void) { } else { CHK_ERR_ASRT(err); IF_VERB(STANDARD) { - std::cout << std::dec << "\tsystem_clock_counter=" << smu.system_clock_counter << '\n'; - std::cout << std::dec << "\ttemperature_edge=" << smu.temperature_edge << '\n'; - std::cout << std::dec << "\ttemperature_hotspot=" << smu.temperature_hotspot << '\n'; - std::cout << std::dec << "\ttemperature_mem=" << smu.temperature_mem << '\n'; - std::cout << std::dec << "\ttemperature_vrgfx=" << smu.temperature_vrgfx << '\n'; - std::cout << std::dec << "\ttemperature_vrsoc=" << smu.temperature_vrsoc << '\n'; - std::cout << std::dec << "\ttemperature_vrmem=" << smu.temperature_vrmem << '\n'; - std::cout << std::dec << "\taverage_gfx_activity=" << smu.average_gfx_activity << '\n'; - std::cout << std::dec << "\taverage_umc_activity=" << smu.average_umc_activity << '\n'; - std::cout << std::dec << "\taverage_mm_activity=" << smu.average_mm_activity << '\n'; - std::cout << std::dec << "\taverage_socket_power=" << smu.average_socket_power << '\n'; - std::cout << std::dec << "\tenergy_accumulator=" << smu.energy_accumulator << '\n'; - std::cout << std::dec << "\taverage_gfxclk_frequency=" << smu.average_gfxclk_frequency << '\n'; - std::cout << std::dec << "\taverage_uclk_frequency=" << smu.average_uclk_frequency << '\n'; - std::cout << std::dec << "\taverage_vclk0_frequency=" << smu.average_vclk0_frequency << '\n'; - std::cout << std::dec << "\taverage_dclk0_frequency=" << smu.average_dclk0_frequency << '\n'; - std::cout << std::dec << "\taverage_vclk1_frequency=" << smu.average_vclk1_frequency << '\n'; - std::cout << std::dec << "\taverage_dclk1_frequency=" << smu.average_dclk1_frequency << '\n'; - std::cout << std::dec << "\tcurrent_gfxclk=" << smu.current_gfxclk << '\n'; - std::cout << std::dec << "\tcurrent_socclk=" << smu.current_socclk << '\n'; - std::cout << std::dec << "\tcurrent_uclk=" << smu.current_uclk << '\n'; - std::cout << std::dec << "\tcurrent_vclk0=" << smu.current_vclk0 << '\n'; - std::cout << std::dec << "\tcurrent_dclk0=" << smu.current_dclk0 << '\n'; - std::cout << std::dec << "\tcurrent_vclk1=" << smu.current_vclk1 << '\n'; - std::cout << std::dec << "\tcurrent_dclk1=" << smu.current_dclk1 << '\n'; - std::cout << std::dec << "\tthrottle_status=" << smu.throttle_status << '\n'; - std::cout << std::dec << "\tcurrent_fan_speed=" << smu.current_fan_speed << '\n'; - std::cout << std::dec << "\tpcie_link_width=" << smu.pcie_link_width << '\n'; - std::cout << std::dec << "\tpcie_link_speed=" << smu.pcie_link_speed << '\n'; - std::cout << std::dec << "\tgfx_activity_acc=" << std::dec << smu.gfx_activity_acc << '\n'; - std::cout << std::dec << "\tmem_activity_acc=" << std::dec << smu.mem_activity_acc << '\n'; - - for (int i = 0; i < RSMI_NUM_HBM_INSTANCES; ++i) { - std::cout << "\ttemperature_hbm[" << i << "]=" << std::dec << smu.temperature_hbm[i] << '\n'; - } - std::cout << "\n"; - std::cout << "\tfirmware_timestamp=" << std::dec << smu.firmware_timestamp << '\n'; - std::cout << "\tvoltage_soc=" << std::dec << smu.voltage_soc << '\n'; - std::cout << "\tvoltage_gfx=" << std::dec << smu.voltage_gfx << '\n'; - std::cout << "\tvoltage_mem=" << std::dec << smu.voltage_mem << '\n'; - std::cout << "\tindep_throttle_status=" << std::dec << smu.indep_throttle_status << '\n'; - std::cout << "\tcurrent_socket_power=" << std::dec << smu.current_socket_power << '\n'; - - for (int i = 0; i < RSMI_MAX_NUM_VCNS; ++i) { - std::cout << "\tvcn_activity[" << i << "]=" << std::dec << smu.vcn_activity[i] << '\n'; - } - std::cout << "\n"; - - for (int i = 0; i < RSMI_MAX_NUM_JPEG_ENGS; ++i) { - std::cout << "\tjpeg_activity[" << i << "]=" << std::dec << smu.jpeg_activity[i] << '\n'; - } - std::cout << "\n"; - - std::cout << "\tgfxclk_lock_status=" << std::dec << smu.gfxclk_lock_status << '\n'; - std::cout << "\txgmi_link_width=" << std::dec << smu.xgmi_link_width << '\n'; - std::cout << "\txgmi_link_speed=" << std::dec << smu.xgmi_link_speed << '\n'; - std::cout << "\tpcie_bandwidth_acc=" << std::dec << smu.pcie_bandwidth_acc << '\n'; - std::cout << "\tpcie_bandwidth_inst=" << std::dec << smu.pcie_bandwidth_inst << '\n'; - std::cout << "\tpcie_l0_to_recov_count_acc=" << std::dec << smu.pcie_l0_to_recov_count_acc << '\n'; - std::cout << "\tpcie_replay_count_acc=" << std::dec << smu.pcie_replay_count_acc << '\n'; - std::cout << "\tpcie_replay_rover_count_acc=" << std::dec << smu.pcie_replay_rover_count_acc << '\n'; - for (int i = 0; i < RSMI_MAX_NUM_XGMI_LINKS; ++i) { - std::cout << "\txgmi_read_data_acc[" << i << "]=" << std::dec << smu.xgmi_read_data_acc[i] << '\n'; - } + std::cout << "METRIC TABLE HEADER:\n"; + std::cout << "structure_size=" << std::dec + << static_cast(smu.common_header.structure_size) << "\n"; + std::cout << "format_revision=" << std::dec + << static_cast(smu.common_header.format_revision) << "\n"; + std::cout << "content_revision=" << std::dec + << static_cast(smu.common_header.content_revision) << "\n"; std::cout << "\n"; - for (int i = 0; i < RSMI_MAX_NUM_XGMI_LINKS; ++i) { - std::cout << "\txgmi_write_data_acc[" << i << "]=" << std::dec << smu.xgmi_write_data_acc[i] << '\n'; - } + std::cout << "TIME STAMPS (ns):\n"; + std::cout << std::dec << "system_clock_counter=" << smu.system_clock_counter << "\n"; + std::cout << "firmware_timestamp (10ns resolution)=" << std::dec << smu.firmware_timestamp + << "\n"; std::cout << "\n"; - for (int i = 0; i < RSMI_MAX_NUM_GFX_CLKS; ++i) { - std::cout << "\tcurrent_gfxclks[" << i << "]=" << std::dec << smu.current_gfxclks[i] << '\n'; - } + std::cout << "TEMPERATURES (C):\n"; + std::cout << std::dec << "temperature_edge= " << smu.temperature_edge << "\n"; + std::cout << std::dec << "temperature_hotspot= " << smu.temperature_hotspot << "\n"; + std::cout << std::dec << "temperature_mem= " << smu.temperature_mem << "\n"; + std::cout << std::dec << "temperature_vrgfx= " << smu.temperature_vrgfx << "\n"; + std::cout << std::dec << "temperature_vrsoc= " << smu.temperature_vrsoc << "\n"; + std::cout << std::dec << "temperature_vrmem= " << smu.temperature_vrmem << "\n"; + std::cout << "temperature_hbm = ["; + std::copy(std::begin(smu.temperature_hbm), + std::end(smu.temperature_hbm), + amd::smi::make_ostream_joiner(&std::cout, ", ")); + std::cout << std::dec << "]\n"; std::cout << "\n"; - for (int i = 0; i < RSMI_MAX_NUM_CLKS; ++i) { - std::cout << "\tcurrent_socclks[" << i << "]=" << std::dec << smu.current_socclks[i] << '\n'; - } + std::cout << "UTILIZATION (%):\n"; + std::cout << std::dec << "average_gfx_activity=" << smu.average_gfx_activity << "\n"; + std::cout << std::dec << "average_umc_activity=" << smu.average_umc_activity << "\n"; + std::cout << std::dec << "average_mm_activity=" << smu.average_mm_activity << "\n"; + std::cout << std::dec << "vcn_activity= ["; + std::copy(std::begin(smu.vcn_activity), + std::end(smu.vcn_activity), + amd::smi::make_ostream_joiner(&std::cout, ", ")); + std::cout << std::dec << "]\n"; std::cout << "\n"; - for (int i = 0; i < RSMI_MAX_NUM_CLKS; ++i) { - std::cout << "\tcurrent_vclk0s[" << i << "]=" << std::dec << smu.current_vclk0s[i] << '\n'; - } + std::cout << std::dec << "jpeg_activity= ["; + std::copy(std::begin(smu.jpeg_activity), + std::end(smu.jpeg_activity), + amd::smi::make_ostream_joiner(&std::cout, ", ")); + std::cout << std::dec << "]\n"; std::cout << "\n"; - for (int i = 0; i < RSMI_MAX_NUM_CLKS; ++i) { - std::cout << "\tcurrent_dclk0s[" << i << "]=" << std::dec << smu.current_dclk0s[i] << '\n'; + std::cout << "POWER (W)/ENERGY (15.259uJ per 1ns):\n"; + std::cout << std::dec << "average_socket_power=" << smu.average_socket_power << "\n"; + std::cout << std::dec << "current_socket_power=" << smu.current_socket_power << "\n"; + std::cout << std::dec << "energy_accumulator=" << smu.energy_accumulator << "\n"; + + std::cout << "\n"; + std::cout << "AVG CLOCKS (MHz):\n"; + std::cout << std::dec << "average_gfxclk_frequency=" << smu.average_gfxclk_frequency + << "\n"; + std::cout << std::dec << "average_gfxclk_frequency=" << smu.average_gfxclk_frequency + << "\n"; + std::cout << std::dec << "average_uclk_frequency=" << smu.average_uclk_frequency << "\n"; + std::cout << std::dec << "average_vclk0_frequency=" << smu.average_vclk0_frequency + << "\n"; + std::cout << std::dec << "average_dclk0_frequency=" << smu.average_dclk0_frequency + << "\n"; + std::cout << std::dec << "average_vclk1_frequency=" << smu.average_vclk1_frequency + << "\n"; + std::cout << std::dec << "average_dclk1_frequency=" << smu.average_dclk1_frequency + << "\n"; + + std::cout << "\n"; + std::cout << "CURRENT CLOCKS (MHz):\n"; + std::cout << std::dec << "current_gfxclk=" << smu.current_gfxclk << "\n"; + std::cout << std::dec << "current_gfxclks= ["; + std::copy(std::begin(smu.current_gfxclks), + std::end(smu.current_gfxclks), + amd::smi::make_ostream_joiner(&std::cout, ", ")); + std::cout << std::dec << "]\n"; + + std::cout << std::dec << "current_socclk=" << smu.current_socclk << "\n"; + std::cout << std::dec << "current_socclks= ["; + std::copy(std::begin(smu.current_socclks), + std::end(smu.current_socclks), + amd::smi::make_ostream_joiner(&std::cout, ", ")); + std::cout << std::dec << "]\n"; + + std::cout << std::dec << "current_uclk=" << smu.current_uclk << "\n"; + std::cout << std::dec << "current_vclk0=" << smu.current_vclk0 << "\n"; + std::cout << std::dec << "current_vclk0s= ["; + std::copy(std::begin(smu.current_vclk0s), + std::end(smu.current_vclk0s), + amd::smi::make_ostream_joiner(&std::cout, ", ")); + std::cout << std::dec << "]\n"; + + std::cout << std::dec << "current_dclk0=" << smu.current_dclk0 << "\n"; + std::cout << std::dec << "current_dclk0s= ["; + std::copy(std::begin(smu.current_dclk0s), + std::end(smu.current_dclk0s), + amd::smi::make_ostream_joiner(&std::cout, ", ")); + std::cout << std::dec << "]\n"; + + std::cout << std::dec << "current_vclk1=" << smu.current_vclk1 << "\n"; + std::cout << std::dec << "current_dclk1=" << smu.current_dclk1 << "\n"; + + std::cout << "\n"; + std::cout << "TROTTLE STATUS:\n"; + std::cout << std::dec << "throttle_status=" << smu.throttle_status << "\n"; + + std::cout << "\n"; + std::cout << "FAN SPEED:\n"; + std::cout << std::dec << "current_fan_speed=" << smu.current_fan_speed << "\n"; + + std::cout << "\n"; + std::cout << "LINK WIDTH (number of lanes) /SPEED (0.1 GT/s):\n"; + std::cout << "pcie_link_width=" << smu.pcie_link_width << "\n"; + std::cout << "pcie_link_speed=" << smu.pcie_link_speed << "\n"; + std::cout << "xgmi_link_width=" << smu.xgmi_link_width << "\n"; + std::cout << "xgmi_link_speed=" << smu.xgmi_link_speed << "\n"; + + std::cout << "\n"; + std::cout << "Utilization Accumulated(%):\n"; + std::cout << "gfx_activity_acc=" << std::dec << smu.gfx_activity_acc << "\n"; + std::cout << "mem_activity_acc=" << std::dec << smu.mem_activity_acc << "\n"; + + std::cout << "\n"; + std::cout << "XGMI ACCUMULATED DATA TRANSFER SIZE (KB):\n"; + std::cout << std::dec << "xgmi_read_data_acc= ["; + std::copy(std::begin(smu.xgmi_read_data_acc), + std::end(smu.xgmi_read_data_acc), + amd::smi::make_ostream_joiner(&std::cout, ", ")); + std::cout << std::dec << "]\n"; + + std::cout << std::dec << "xgmi_write_data_acc= ["; + std::copy(std::begin(smu.xgmi_write_data_acc), + std::end(smu.xgmi_write_data_acc), + amd::smi::make_ostream_joiner(&std::cout, ", ")); + std::cout << std::dec << "]\n"; + + // Voltage (mV) + std::cout << "voltage_soc = " << std::dec << smu.voltage_soc << "\n"; + std::cout << "voltage_gfx = " << std::dec << smu.voltage_gfx << "\n"; + std::cout << "voltage_mem = " << std::dec << smu.voltage_mem << "\n"; + + std::cout << "indep_throttle_status = " << std::dec << smu.indep_throttle_status << "\n"; + + // Clock Lock Status. Each bit corresponds to clock instance + std::cout << "gfxclk_lock_status (in hex) = " << std::hex + << smu.gfxclk_lock_status << std::dec <<"\n"; + + // Bandwidth (GB/sec) + std::cout << "pcie_bandwidth_acc=" << std::dec << smu.pcie_bandwidth_acc << "\n"; + std::cout << "pcie_bandwidth_inst=" << std::dec << smu.pcie_bandwidth_inst << "\n"; + + // Counts + std::cout << "pcie_l0_to_recov_count_acc= " << std::dec << smu.pcie_l0_to_recov_count_acc + << "\n"; + std::cout << "pcie_replay_count_acc= " << std::dec << smu.pcie_replay_count_acc << "\n"; + std::cout << "pcie_replay_rover_count_acc= " << std::dec + << smu.pcie_replay_rover_count_acc << "\n"; + std::cout << "pcie_nak_sent_count_acc= " << std::dec << smu.pcie_nak_sent_count_acc + << "\n"; + std::cout << "pcie_nak_rcvd_count_acc= " << std::dec << smu.pcie_nak_rcvd_count_acc + << "\n"; + + // PCIE other end recovery counter + std::cout << "pcie_lc_perf_other_end_recovery = " + << std::dec << smu.pcie_lc_perf_other_end_recovery << "\n"; + + // Accumulation cycle counter + // Accumulated throttler residencies + std::cout << "\n"; + std::cout << "RESIDENCY ACCUMULATION / COUNTER:\n"; + std::cout << "accumulation_counter = " << std::dec << smu.accumulation_counter << "\n"; + std::cout << "prochot_residency_acc = " << std::dec << smu.prochot_residency_acc << "\n"; + std::cout << "ppt_residency_acc = " << std::dec << smu.ppt_residency_acc << "\n"; + std::cout << "socket_thm_residency_acc = " << std::dec << smu.socket_thm_residency_acc + << "\n"; + std::cout << "vr_thm_residency_acc = " << std::dec << smu.vr_thm_residency_acc + << "\n"; + std::cout << "hbm_thm_residency_acc = " << std::dec << smu.hbm_thm_residency_acc << "\n"; + + // Number of current partitions + std::cout << "num_partition = " << std::dec << smu.num_partition << "\n"; + + + std::cout << std::dec << "xcp_stats.gfx_busy_inst = \n"; + auto xcp = 0; + for (auto& row : smu.xcp_stats) { + std::cout << "XCP[" << xcp << "] = " << "[ "; + std::copy(std::begin(row.gfx_busy_inst), + std::end(row.gfx_busy_inst), + amd::smi::make_ostream_joiner(&std::cout, ", ")); + std::cout << " ]\n"; + xcp++; } + xcp = 0; + std::cout << std::dec << "xcp_stats.jpeg_busy = \n"; + for (auto& row : smu.xcp_stats) { + std::cout << "XCP[" << xcp << "] = " << "[ "; + std::copy(std::begin(row.jpeg_busy), + std::end(row.jpeg_busy), + amd::smi::make_ostream_joiner(&std::cout, ", ")); + std::cout << " ]\n"; + xcp++; + } + + xcp = 0; + std::cout << std::dec << "xcp_stats.vcn_busy = \n"; + for (auto& row : smu.xcp_stats) { + std::cout << "XCP[" << xcp << "] = " << "[ "; + std::copy(std::begin(row.vcn_busy), + std::end(row.vcn_busy), + amd::smi::make_ostream_joiner(&std::cout, ", ")); + std::cout << " ]\n"; + xcp++; + } + + xcp = 0; + std::cout << std::dec << "xcp_stats.gfx_busy_acc = \n"; + for (auto& row : smu.xcp_stats) { + std::cout << "XCP[" << xcp << "] = " << "[ "; + std::copy(std::begin(row.gfx_busy_acc), + std::end(row.gfx_busy_acc), + amd::smi::make_ostream_joiner(&std::cout, ", ")); + std::cout << " ]\n"; + xcp++; + } std::cout << "\n\n"; std::cout << "\t ** -> Checking metrics with constant changes ** " << "\n"; @@ -256,17 +366,20 @@ void TestGpuMetricsRead::Run(void) { rsmi_gpu_metrics_t gpu_metrics_check; for (auto idx = uint16_t(1); idx <= kMAX_ITER_TEST; ++idx) { rsmi_dev_gpu_metrics_info_get(i, &gpu_metrics_check); - std::cout << "\t\t -> firmware_timestamp [" << idx << "/" << kMAX_ITER_TEST << "]: " << gpu_metrics_check.firmware_timestamp << "\n"; + std::cout << "\t\t -> firmware_timestamp [" << idx << "/" << kMAX_ITER_TEST << "]: " + << gpu_metrics_check.firmware_timestamp << "\n"; } std::cout << "\n"; for (auto idx = uint16_t(1); idx <= kMAX_ITER_TEST; ++idx) { rsmi_dev_gpu_metrics_info_get(i, &gpu_metrics_check); - std::cout << "\t\t -> system_clock_counter [" << idx << "/" << kMAX_ITER_TEST << "]: " << gpu_metrics_check.system_clock_counter << "\n"; + std::cout << "\t\t -> system_clock_counter [" << idx << "/" << kMAX_ITER_TEST << "]: " + << gpu_metrics_check.system_clock_counter << "\n"; } std::cout << "\n"; - std::cout << " ** Note: Values MAX'ed out (UINTX MAX are unsupported for the version in question) ** " << "\n\n"; + std::cout << " ** Note: Values MAX'ed out " + << "(UINTX MAX are unsupported for the version in question) ** " << "\n\n"; } } diff --git a/projects/rocm-smi-lib/tests/rocm_smi_test/functional/measure_api_execution_time.cc b/projects/rocm-smi-lib/tests/rocm_smi_test/functional/measure_api_execution_time.cc index 6f83e057ef..95e6e45f65 100644 --- a/projects/rocm-smi-lib/tests/rocm_smi_test/functional/measure_api_execution_time.cc +++ b/projects/rocm-smi-lib/tests/rocm_smi_test/functional/measure_api_execution_time.cc @@ -53,6 +53,7 @@ #include "rocm_smi/rocm_smi.h" #include "rocm_smi_test/functional/measure_api_execution_time.h" #include "rocm_smi_test/test_common.h" +#include "rocm_smi/rocm_smi_utils.h" TestMeasureApiExecutionTime::TestMeasureApiExecutionTime() : TestBase() { @@ -89,8 +90,31 @@ void TestMeasureApiExecutionTime::Run(void) { rsmi_temperature_metric_t met = RSMI_TEMP_CURRENT; rsmi_status_t ret; float repeat = 300.0; - constexpr uint32_t kFAN_SPEED_ELAPSED_MS_BASE = (1000); - constexpr uint32_t kMETRICS_ELAPSED_MS_BASE = (1500); + constexpr float kFAN_SPEED_ELAPSED_MICROSEC_BASE = (1000); + /** + * gpu_metrics can only refresh every 1000 microseconds (1 millisecs) due to FW + * + * We have additional processing time (each read() -> fread() ~ costs 900 microseconds). + * We need to read 2x: + * 1) reading metric's header to check support (~900 microseconds) + * 2) read full metric based on defined structure (~900 microseconds) + * 3) Setup backwards compatiblity (~100 microseconds) + * 4) Put data into structures (~100 microseconds) + * 5) Pass to public structure (~100 microseconds) + * --------------------------- + * ~2100 worst case + * + * Note: performance of fread/mmap/read + * https://github.com/nurettn/c-read-vs-mmap-vs-fread + * + * Possible improvments ideas: + * a) Initize "N/A" / Max UINT only for non-backwards comptable public struct + * or arrays + * b) Directly put data into public structure - this skips other copy/fill + * procedures + * c) Expirement with other file reading options + **/ + constexpr float kMETRICS_ELAPSED_MICROSEC_BASE = (2100); bool skip = false; TestBase::Run(); @@ -107,91 +131,125 @@ void TestMeasureApiExecutionTime::Run(void) { for (uint32_t dv_ind = 0; dv_ind < num_monitor_devs(); ++dv_ind) { PrintDeviceHeader(dv_ind); - //test execution time for rsmi_dev_fan_speed_get + // test execution time for rsmi_dev_fan_speed_get auto start = std::chrono::high_resolution_clock::now(); - for (int i=0; i < static_cast(repeat); ++i){ + for (int i=0; i < static_cast(repeat); ++i) { ret = rsmi_dev_fan_speed_get(dv_ind, 0, &val_i64); - } auto stop = std::chrono::high_resolution_clock::now(); auto duration = std::chrono::duration_cast (stop - start); - if (ret != RSMI_STATUS_SUCCESS){ + std::cout << "\n\trsmi_dev_fan_speed_get returned: " + << amd::smi::getRSMIStatusString(ret) << "\n"; + if (ret != RSMI_STATUS_SUCCESS) { skip = true; } std::cout << std:: endl; + // Expected performance: (stop - start) over all iterations [in microseconds] + // == (expected microseconds * # of iterations) + if (!skip) { - std::cout << "\trsmi_dev_fan_speed_get execution time: " << - (static_cast(duration.count()) / repeat) << " microseconds" << std::endl; - EXPECT_LT(duration.count(), (kFAN_SPEED_ELAPSED_MS_BASE * repeat)); + std::cout << "\trsmi_dev_fan_speed_get() total execution time: " + << std::to_string((static_cast(duration.count()))) + << " microseconds, expected < " + << std::to_string((static_cast(kFAN_SPEED_ELAPSED_MICROSEC_BASE) * repeat)) + << " microseconds" << std::endl; + std::cout << "\trsmi_dev_fan_speed_get() average execution time: " + << std::to_string(duration.count()/repeat) << " microseconds" << std::endl; + EXPECT_LT(duration.count(), static_cast(kFAN_SPEED_ELAPSED_MICROSEC_BASE) * repeat); } skip = false; - //test execution time for rsmi_dev_temp_metric_get + // test execution time for rsmi_dev_temp_metric_get start = std::chrono::high_resolution_clock::now(); - for (int i=0; i < static_cast(repeat); ++i){ + for (int i=0; i < static_cast(repeat); ++i) { ret = rsmi_dev_temp_metric_get(dv_ind, 0, met, &val_i64); } stop = std::chrono::high_resolution_clock::now(); duration = std::chrono::duration_cast (stop - start); - if (ret != RSMI_STATUS_SUCCESS){ + std::cout << "\n\trsmi_dev_temp_metric_get returned: " + << amd::smi::getRSMIStatusString(ret) << "\n"; + if (ret != RSMI_STATUS_SUCCESS) { skip = true; } if (!skip) { - std::cout << "\trsmi_dev_temp_metric_get execution time: " << - (static_cast(duration.count()) / repeat) << " microseconds" << std::endl; - EXPECT_LT(duration.count(), (kMETRICS_ELAPSED_MS_BASE * repeat)); + std::cout << "\trsmi_dev_temp_metric_get() total execution time: " + << std::to_string((static_cast(duration.count()))) + << " microseconds, expected < " + << std::to_string((static_cast(kMETRICS_ELAPSED_MICROSEC_BASE) * repeat)) + << " microseconds" << std::endl; + std::cout << "\trsmi_dev_temp_metric_get() average execution time: " + << std::to_string(duration.count()/repeat) << " microseconds" << std::endl; + EXPECT_LT(duration.count(), (static_cast(kMETRICS_ELAPSED_MICROSEC_BASE) * repeat)); } skip = false; - //test execution time for rsmi_dev_gpu_metrics_info_get + // test execution time for rsmi_dev_gpu_metrics_info_get start = std::chrono::high_resolution_clock::now(); - for (int i=0; i < static_cast(repeat); ++i){ + for (int i=0; i < static_cast(repeat); ++i) { ret = rsmi_dev_gpu_metrics_info_get(dv_ind, &smu); } stop = std::chrono::high_resolution_clock::now(); duration = std::chrono::duration_cast - (stop - start) ; + (stop - start); - if (ret != RSMI_STATUS_SUCCESS){ + std::cout << "\n\trsmi_dev_gpu_metrics_info_get returned: " + << amd::smi::getRSMIStatusString(ret) << "\n"; + if (ret != RSMI_STATUS_SUCCESS) { skip = true; } if (!skip) { - std::cout << "\trsmi_dev_gpu_metrics_info_get execution time: " << - (static_cast(duration.count()) / repeat ) << " microseconds" << std::endl; - EXPECT_LT(duration.count(), (kMETRICS_ELAPSED_MS_BASE * repeat)); + std::cout << "\trsmi_dev_gpu_metrics_info_get() total execution time: " + << std::to_string(static_cast(duration.count())) + << " microseconds, expected < " + << std::to_string((kMETRICS_ELAPSED_MICROSEC_BASE * repeat)) + << " microseconds" << std::endl; + std::cout << "\trsmi_dev_gpu_metrics_info_get() average execution time: " + << std::to_string(duration.count()/repeat) << " microseconds" << std::endl; + EXPECT_LT(static_cast(duration.count()), + static_cast(kMETRICS_ELAPSED_MICROSEC_BASE) * repeat); } skip = false; auto val_ui16 = static_cast(0); auto status_code(rsmi_status_t::RSMI_STATUS_SUCCESS); start = std::chrono::high_resolution_clock::now(); - for (int i=0; i < static_cast(repeat); ++i){ + for (int i=0; i < static_cast(repeat); ++i) { status_code = rsmi_dev_metrics_xcd_counter_get(dv_ind, &val_ui16); } stop = std::chrono::high_resolution_clock::now(); duration = std::chrono::duration_cast(stop - start); - if (status_code != rsmi_status_t::RSMI_STATUS_SUCCESS){ + std::cout << "\n\tsmi_dev_metrics_xcd_counter_get returned: " + << amd::smi::getRSMIStatusString(ret) << "\n"; + if (status_code != rsmi_status_t::RSMI_STATUS_SUCCESS) { skip = true; } if (!skip) { - std::cout << "\trsmi_dev_metrics_xcd_counter_get() execution time: " - << (static_cast(duration.count()) / repeat) << " microseconds" << std::endl; - EXPECT_LT(duration.count(), (kMETRICS_ELAPSED_MS_BASE * repeat)); + std::cout << "\trsmi_dev_metrics_xcd_counter_get() total execution time: " + << std::to_string((static_cast(duration.count()))) + << " microseconds, expected < " + << std::to_string((static_cast(kMETRICS_ELAPSED_MICROSEC_BASE) * repeat)) + << " microseconds" << std::endl; + std::cout << "\trsmi_dev_metrics_xcd_counter_get() average execution time: " + << std::to_string(duration.count()/repeat) << " microseconds" << std::endl; + EXPECT_LT(duration.count(), static_cast(kMETRICS_ELAPSED_MICROSEC_BASE) * repeat); } skip = false; } std::cout.precision(prev); auto test_stop = std::chrono::high_resolution_clock::now(); - auto test_duration = std::chrono::duration_cast(test_stop - test_start); + auto test_duration = std::chrono::duration_cast( + test_stop - test_start); - std::cout << "\n" << "============================================================================" << "\n"; + std::cout << "\n" + << "============================================================================" << "\n"; std::cout << " Total execution time (All APIs): " - << (static_cast(test_duration.count()) / repeat) << " microseconds" << "\n"; - std::cout << "============================================================================" << "\n"; + << (test_duration.count()) << " microseconds" << "\n"; + std::cout + << "============================================================================" << "\n"; }