From 5b3a8d5075adae72dc331ee933d591383e8de056 Mon Sep 17 00:00:00 2001 From: Nicholas Curtis Date: Tue, 6 Jun 2023 11:01:37 -0400 Subject: [PATCH] Incorporate review comments Signed-off-by: Nicholas Curtis --- src/docs/analysis.md | 16 ++++++++-------- src/docs/conf.py | 4 ++++ src/docs/getting_started.md | 20 ++++++++++++-------- src/docs/high_level_design.md | 4 ++-- src/docs/installation.md | 14 +++++++++----- src/docs/introduction.md | 9 ++++----- src/docs/profiling.md | 15 +++++++++------ src/parser.py | 2 +- 8 files changed, 49 insertions(+), 35 deletions(-) diff --git a/src/docs/analysis.md b/src/docs/analysis.md index 2321ddabd3..9feff1f64b 100644 --- a/src/docs/analysis.md +++ b/src/docs/analysis.md @@ -26,7 +26,7 @@ Run `omniperf analyze -h` for more details. ### Recommended workflow 1) Do a comprehensive analysis with Omniperf CLI at the beginning. -```shell +```shell-session $ omniperf analyze -p workloads/vcopy/mi200/ -------- @@ -108,7 +108,7 @@ Analyze .... ``` 2. Use `--list-metrics` to generate a list of availible metrics for inspection - ```shell + ```shell-session $ omniperf analyze -p workloads/vcopy/mi200/ --list-metrics gfx90a ╒═════════╤═════════════════════════════╕ │ │ Metric │ @@ -172,7 +172,7 @@ $ omniperf analyze -p workloads/vcopy/mi200/ --list-metrics gfx90a ... ``` 2. Choose your own customized subset of metrics with `-b` (a.k.a. `--metric`), or build your own config following [config_template](https://github.com/AMDResearch/omniperf/blob/main/src/omniperf_analyze/configs/panel_config_template.yaml). Below we'll inspect block 2 (a.k.a. System Speed-of-Light). -```shell +```shell-session $ omniperf analyze -p workloads/vcopy/mi200/ -b 2 -------- Analyze @@ -286,7 +286,7 @@ Analyze - Filter kernels First, list the top kernels in your application using `--list-kernels`. - ```shell + ```shell-session $ omniperf analyze -p workloads/vcopy/mi200/ --list-kernels -------- @@ -306,7 +306,7 @@ Analyze Second, select the index of the kernel you'd like to filter (i.e. __vecCopy(double*, double*, double*, int, int) [clone .kd]__ at index __0__). Then, use this index to apply the filter via `-k/--kernels`. - ```shell + ```shell-session $ omniperf -p workloads/vcopy/mi200/ -k 0 -------- @@ -372,7 +372,7 @@ See [FAQ](https://amdresearch.github.io/omniperf/faq.html) for more details on S To launch the standalone GUI, include the `--gui` flag with your desired analysis command. For example: -```bash +```shell-session $ omniperf analyze -p workloads/vcopy/mi200/ --gui -------- @@ -499,7 +499,7 @@ e.g., omniperf_asw_vcopy_mi200. Below is the sample command to import the *vcopy* profiling data. -```shell +```shell-session $ omniperf database --help ROC Profiler: /usr/bin/rocprof @@ -544,7 +544,7 @@ Connection Options: ``` **omniperf import for vcopy:** -```shell +```shell-session $ omniperf database --import -H pavii1 -u temp -t asw -w workloads/vcopy/mi200/ ROC Profiler: /usr/bin/rocprof diff --git a/src/docs/conf.py b/src/docs/conf.py index 48d4c5596f..b659553f9a 100644 --- a/src/docs/conf.py +++ b/src/docs/conf.py @@ -53,6 +53,10 @@ extensions = [ ] myst_heading_anchors = 2 +# enable replacement of (tm) & friends +myst_enable_extensions = [ + "replacements" +] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] diff --git a/src/docs/getting_started.md b/src/docs/getting_started.md index c75bda9abe..80ae888f06 100644 --- a/src/docs/getting_started.md +++ b/src/docs/getting_started.md @@ -10,24 +10,28 @@ 1. **Launch & Profile the target application with the command line profiler** - The command line profiler launches the target application, calls the rocProfiler API, and collects profile results for the specified kernels, dispatches, and/or ipblock’s. + The command line profiler launches the target application, calls the rocProfiler API, and collects profile results for the specified kernels, dispatches, and/or IP blocks. If not specified, Omniperf will default to collecting all available counters for all kernels/dispatches launched by the user's executable. - To collect the default set of data for all kernels in the target application, launch: + To collect the default set of data for all kernels in the target application, launch, e.g.: ```shell - $ omniperf profile -n vcopy -- ./vcopy 1048576 256 + $ omniperf profile -n vcopy_data -- ./vcopy 1048576 256 ``` - The app runs, each kernel is launched, and profiling results are generated. By default, results are written to ./workloads/\. To collect all requested profile information, it may be required to replay kernels multiple times. + The app runs, each kernel is launched, and profiling results are generated. By default, results are written to (e.g.,) ./workloads/vcopy_data (configurable via the `-n` argument). To collect all requested profile information, it may be required to replay kernels multiple times. 2. **Customize data collection** - Options are available to specify for which kernels data should be collected. - `-k`/`--kernel` enables filtering kernels by name. `-d`/`--dispatch` enables filtering based on dispatch ID. `-b`/`--ipblocks` enables profiling on one or more IP Block(s). + Options are available to specify for which kernels/metrics data should be collected. + Note that filtering can be applied either in the profiling or analysis stage, however filtering at during profiling collection will often speed up your overall profiling run time. - To view available metrics by IP Block you can always use `--list-metrics` to view a list of all available metrics organized by IP Block. + Some common filters include: + + - `-k`/`--kernel` enables filtering kernels by name. `-d`/`--dispatch` enables filtering based on dispatch ID + - `-b`/`--ipblocks` enables collects metrics for only the specified (one or more) IP Blocks. + + To view available metrics by IP Block you can use the `--list-metrics` argument to view a list of all available metrics organized by IP Block. ```shell $ omniperf analyze --list-metrics ``` - Note that filtering can also be applied after the fact, at the analysis stage, however filtering at the profiling level will often speed up your overall profiling run time. 3. **Analyze at the command line** diff --git a/src/docs/high_level_design.md b/src/docs/high_level_design.md index 28c09ff129..6168b7ace9 100644 --- a/src/docs/high_level_design.md +++ b/src/docs/high_level_design.md @@ -8,10 +8,10 @@ The [Omniperf](https://github.com/AMDResearch/omniperf) Tool is architecturally composed of three major components, as shown in the following figure. -- **Omniperf Profiling**: Acquire raw performance counters via application replay based on the [ROC Profiler](https://github.com/ROCm-Developer-Tools/rocprofiler). A set of MI200 specific micro benchmarks are also run to acquire the hierarchical roofline data. +- **Omniperf Profiling**: Acquire raw performance counters via application replay based on the [ROC Profiler](https://github.com/ROCm-Developer-Tools/rocprofiler). The counters are stored in a comma-seperated value, for further analyis. A set of MI200 specific micro benchmarks are also run to acquire the hierarchical roofline data. The roofline model is not available on earlier accelerators. - **Omniperf Grafana Analyzer**: - - *Grafana database import*: All raw performance counters are imported into the backend MongoDB database for Grafana GUI analysis and visualization. + - *Grafana database import*: All raw performance counters are imported into the backend MongoDB database for Grafana GUI analysis and visualization. Compatibility of previously generated data between Omniperf versions is not necessarily guarenteed. - *Grafana GUI Analyzer*: A Grafana dashboard is designed to retrieve the raw counters info from the backend database. It also creates the relevant performance metrics and visualization. - **Omniperf Standalone GUI Analyzer**: A standalone GUI is provided to enable performance analysis without importing data into the backend database. diff --git a/src/docs/installation.md b/src/docs/installation.md index ee2489710c..e550669b7e 100644 --- a/src/docs/installation.md +++ b/src/docs/installation.md @@ -109,7 +109,7 @@ ROC Profiler: /opt/rocm-5.1.0/bin/rocprof omniperf (v{__VERSION__}) ``` -```{tip} Sites relying on an Lmod Python module locally may wish to +```{tip} Users relying on an Lmod Python module locally may wish to customize the resulting Omniperf modulefile post-installation to include additional module dependencies. ``` @@ -129,8 +129,9 @@ export PYTHONPATH=$INSTALL_DIR/python-libs Omniperf relies on a rocprof binary during the profiling process. Normally the path to this binary will be detected -automatically, but it can also be overridden via the use of an -optional `ROCPROF` environment variable. +automatically, but it can also be overridden via the setting the +optional `ROCPROF` environment variable to the path of the binary the user +wishes to use instead. @@ -162,9 +163,12 @@ Omniperf uses [mongoimport](https://www.mongodb.com/docs/database-tools/mongoimp $ wget https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu2004-x86_64-100.6.1.deb $ sudo apt install ./mongodb-database-tools-ubuntu2004-x86_64-100.6.1.deb ``` -> Find install for alternative distros [here](https://www.mongodb.com/download-center/database-tools/releases/archive) +> Installation instructions for alternative distributions can be found [here](https://www.mongodb.com/download-center/database-tools/releases/archive) + +### Persistent Storage + +The user may also choose to bind MongoDB to a directory on the host OS to create a local backup in case of a crash or reset: -### Persist Storage ```bash $ sudo mkdir -p /usr/local/persist && cd /usr/local/persist/ $ sudo mkdir -p grafana-storage mongodb diff --git a/src/docs/introduction.md b/src/docs/introduction.md index 436146db8b..6b39d4088a 100644 --- a/src/docs/introduction.md +++ b/src/docs/introduction.md @@ -10,17 +10,17 @@ ## Scope -MI Performance Profiler ([Omniperf](https://github.com/AMDResearch/omniperf)) is a system performance profiling tool for Machine Learning/HPC workloads running on AMD MI GPUs. It is currently built on top of the [ROC Profiler](https://github.com/ROCm-Developer-Tools/rocprofiler) to monitor hardware performance counters. The Omniperf tool primarily targets MI100 and MI200 silicon. Development is in progress to support MI300 and NAVI GPUs. +MI Performance Profiler ([Omniperf](https://github.com/AMDResearch/omniperf)) is a system performance profiling tool for Machine Learning/HPC workloads running on AMD Instinct (tm) Accelerators. It is currently built on top of the [ROC Profiler](https://github.com/ROCm-Developer-Tools/rocprofiler) to monitor hardware performance counters. The Omniperf tool primarily targets accelerators in the MI100 and MI200 families. Development is in progress to support MI300 and Radeon (tm) RDNA (tm) GPUs. ## Features -The Omniperf tool performs system profiling based on all approved hardware counters for MI200. It provides high level performance analysis features including System Speed-of-Light, IP block Speed-of-Light, Memory Chart Analysis, Roofline Analysis, Baseline Comparisons, and more... +The Omniperf tool performs system profiling based on all available hardware counters for the target accelerator. It provides high level performance analysis features including System Speed-of-Light, IP block Speed-of-Light, Memory Chart Analysis, Roofline Analysis, Baseline Comparisons, and more... Both command line analysis and GUI analysis are supported. Detailed Feature List: -- MI200 support - MI100 support +- MI200 support - Standalone GUI Analyzer - Grafana/MongoDB GUI Analyzer - Dispatch Filtering @@ -50,8 +50,7 @@ Detailed Feature List: | Platform | Status | | :------- | :------------- | -| Vega 20 | No | -| MI50 | No | +| Vega 20 (MI-50/60) | No | | MI100 | Supported | | MI200 | Supported | | MI300 | In development | diff --git a/src/docs/profiling.md b/src/docs/profiling.md index 6776097c92..1a95477758 100644 --- a/src/docs/profiling.md +++ b/src/docs/profiling.md @@ -19,7 +19,7 @@ the MI200 platform. ## Workload Compilation **vcopy compilation:** -```shell +```shell-session $ hipcc vcopy.cpp -o vcopy $ ls vcopy vcopy.cpp @@ -40,7 +40,7 @@ Releasing CPU memory The *omniperf* script, availible through the [Omniperf](https://github.com/AMDResearch/omniperf) repository, is used to aquire all necessary perfmon data through analysis of compute workloads. **omniperf help:** -```shell +```shell-session $ omniperf profile --help ROC Profiler: /usr/bin/rocprof @@ -56,7 +56,7 @@ Examples: omniperf profile -n vcopy_all -- ./vcopy 1048576 256 - omniperf profile -n vcopy_SPI_TD -b SQ TCC -- ./vcopy 1048576 256 + omniperf profile -n vcopy_SPI_TCC -b SQ TCC -- ./vcopy 1048576 256 omniperf profile -n vcopy_kernel -k vecCopy -- ./vcopy 1048576 256 @@ -111,7 +111,7 @@ Standalone Roofline Options: The following sample command profiles the *vcopy* workload. **vcopy profiling:** -```shell +```shell-session $ omniperf profile --name vcopy -- ./vcopy 1048576 256 Resolving rocprof ROC Profiler: /usr/bin/rocprof @@ -206,7 +206,10 @@ Peak MFMA IOPs (I8), GPU ID: 1, workgroupSize:256, workgroups:16384, experiments ``` You'll notice two stages in *default* Omniperf profiling. The first stage collects all the counters needed for Omniperf analysis (omitting any filters you've provided). The second stage collects data for the roofline analysis (this stage can be disabled using `--no-roof`) -At the end of the profiling, all resulting csv files should be located in the SOC specific target directory, e.g., mi200. +At the end of the profiling, all resulting csv files should be located in a SOC specific target directory, e.g.: + - "mi200" for the AMD Instinct (tm) MI-200 family of accelerators + - "mi100" for the AMD Instinct (tm) MI-100 family of accelerators +etc. The SOC names are generated as a part of Omniperf, and do not necessarily distinguish between different accelerators in the same family (e.g., an AMD Instinct (tm) MI-210 vs an MI-250) > Note: Additionally, you'll notice a few extra files. An SoC parameters file, *sysinfo.csv*, is created to reflect the target device settings. All profiling output is stored in *log.txt*. Roofline specific benchmark results are stored in *roofline.csv*. @@ -316,7 +319,7 @@ ROCProfiler: input from "/tmp/rpl_data_230411_170300_29696/input0.xml" #### Dispatch Filtering The following example demonstrates profiling on selected dispatches: -```shell +```shell-session $ omniperf profile --name vcopy -d 0 -- ./vcopy 1048576 256 Resolving rocprof ROC Profiler: /usr/bin/rocprof diff --git a/src/parser.py b/src/parser.py index da018ba944..9d6dd8f6f2 100644 --- a/src/parser.py +++ b/src/parser.py @@ -66,7 +66,7 @@ def parse(my_parser): \n\n------------------------------------------------------------------------------- \nExamples: \n\tomniperf profile -n vcopy_all -- ./vcopy 1048576 256 - \n\tomniperf profile -n vcopy_SPI_TD -b SQ TCC -- ./vcopy 1048576 256 + \n\tomniperf profile -n vcopy_SPI_TCC -b SQ TCC -- ./vcopy 1048576 256 \n\tomniperf profile -n vcopy_kernel -k vecCopy -- ./vcopy 1048576 256 \n\tomniperf profile -n vcopy_disp -d 0 -- ./vcopy 1048576 256 \n\tomniperf profile -n vcopy_roof --roof-only -- ./vcopy 1048576 256