Support MI 350 profiling (#632)
* Add MI 350 hardware information
* Refactor MI GPU YAML file and corresponding interface
* Add SoC file for gfx950 architecture
* Add analysis report configs for MI 350 containing existing metrics
* Add placeholder None valued metrics for previous architectures to make
baseline comparison work
* Enable testing on MI 350
* Analysis config metric changes
- SPI changes
- Update metric formula for default SPI pipe counter
- Use efficiently collected pipe wise SPI counters
- Add SPI Wave Occupancy
- Add Scheduler-Pipe Wave Utilization
- Update formula for VGPR Writes
- Add Scheduler-Pipe FIFO Full Rate
- CPC changes
- Add CPC SYNC FIFO Full Rate
- Add CPC CANE Stall Rate
- Add CPC ADC Utilization
- SQ changes
- Add VALU co-issue efficiency
- Add F6F4 datatype metrics
- Update formula for total FLOPs by adding F6F4 counters
- Add LDS STORE / LOAD / ATOMIC metrics
- Add LDS STORE / LOAD / ATOMIC bandwidth
- Add LDS FIFO and TA ADDR / CMD / DATA FIFO full rates
* Collect TCP_TCP_LATENCY_sum only for gfx950 (MI 350)
* Do not inject SQ_ACCUM_PREV_HIRES unnecesarily
* Do not hardcode memory and shader clock speeds
* Write num_hbm_channels to sysinfo.csv instead of hbm_bw while profiling
* Move generate sysinfo.csv to pre processing step of profiling
* Add warnings to use --specs-correction for missing sysinfo.csv values during analysis phase
* Update CHANGELOG
* Analysis phase warning to use --specs-correction when needed
[ROCm/rocprofiler-compute commit: f9aa7be97c]
Этот коммит содержится в:
коммит произвёл
GitHub
родитель
1273a5e2a9
Коммит
27585a8a2b
@@ -120,7 +120,7 @@ def discrete_background_color_bins(df, n_bins=5, columns="all"):
|
||||
####################
|
||||
# GRAPHICAL ELEMENTS
|
||||
####################
|
||||
def build_bar_chart(display_df, table_config, barchart_elements, norm_filt, hbm_bw):
|
||||
def build_bar_chart(display_df, table_config, barchart_elements, norm_filt):
|
||||
"""
|
||||
Read data into a bar chart. ID will determine which subtype of barchart.
|
||||
"""
|
||||
@@ -214,6 +214,9 @@ def build_bar_chart(display_df, table_config, barchart_elements, norm_filt, hbm_
|
||||
orientation="h",
|
||||
).update_xaxes(range=[0, 110], ticks="inside", title="%")
|
||||
) # append first % chart
|
||||
hbm_bw = float(
|
||||
display_df[display_df["Metric"] == "HBM Bandwidth"]["Avg"].iloc[0]
|
||||
)
|
||||
d_figs.append(
|
||||
px.bar(
|
||||
display_df[display_df["Unit"] == "Gb/s"],
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
import os
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Dict
|
||||
|
||||
import yaml
|
||||
|
||||
@@ -13,14 +11,20 @@ MI50 = 0
|
||||
MI100 = 1
|
||||
MI200 = 2
|
||||
MI300 = 3
|
||||
MI350 = 4
|
||||
|
||||
MI_CONSTANS = {MI50: "mi50", MI100: "mi100", MI200: "mi200", MI300: "mi300"}
|
||||
MI_CONSTANS = {
|
||||
MI50: "mi50",
|
||||
MI100: "mi100",
|
||||
MI200: "mi200",
|
||||
MI300: "mi300",
|
||||
MI350: "mi350",
|
||||
}
|
||||
|
||||
gpu_series_dict = {} # key: gpu arch
|
||||
gpu_model_dict = {} # key: gpu_arch
|
||||
mi300_num_xcds_dict = {} # key: gpu model
|
||||
mi300_nps_dict = {} # key: gpu model
|
||||
mi300_chip_id_dict = {} # key: chip id (int)
|
||||
num_xcds_dict = {} # key: gpu model
|
||||
chip_id_dict = {} # key: chip id (int)
|
||||
|
||||
|
||||
# ----------------------------
|
||||
@@ -60,10 +64,9 @@ def parse_mi_gpu_spec():
|
||||
MI GPUs
|
||||
|-- series
|
||||
|-- architecture (list)
|
||||
|-- models
|
||||
|-- chip_ids
|
||||
|-- mi300_arch
|
||||
|-- partition_mode
|
||||
|-- gpu model
|
||||
|-- chip_ids
|
||||
|-- partition_mode
|
||||
"""
|
||||
|
||||
current_dir = os.path.dirname(__file__)
|
||||
@@ -71,61 +74,26 @@ def parse_mi_gpu_spec():
|
||||
|
||||
# Load the YAML data
|
||||
yaml_data = load_yaml(yaml_file_path)
|
||||
mi300_models_dict = {}
|
||||
|
||||
for mi_index, mi_series in MI_CONSTANS.items():
|
||||
if mi_series != MI_CONSTANS[MI300]:
|
||||
console_debug("[parse_mi_gpu_spec] Processing series: %s" % mi_series)
|
||||
for key, value in yaml_data.items():
|
||||
# parse out gpu series and gpu model information for mi50, 100, 200
|
||||
curr_gpu_arch = value[mi_index]["gpu_archs"][0]["gpu_arch"]
|
||||
gpu_series_dict[curr_gpu_arch] = mi_series
|
||||
gpu_model_dict[curr_gpu_arch] = []
|
||||
for models in value[mi_index]["gpu_archs"][0]["models"]:
|
||||
gpu_model_dict[curr_gpu_arch].append(models["gpu_model"])
|
||||
elif mi_series == MI_CONSTANS[MI300]:
|
||||
# MI300 requires specific processing
|
||||
for key, value in yaml_data.items():
|
||||
mi300_gpu_archs_list = []
|
||||
# NOTE: only MI300 have multiple architectures
|
||||
for archs in value[MI300]["gpu_archs"]:
|
||||
curr_gpu_arch = archs["gpu_arch"]
|
||||
mi300_gpu_archs_list.append(curr_gpu_arch)
|
||||
gpu_series_dict[curr_gpu_arch] = mi_series
|
||||
|
||||
for idx, arch in enumerate(mi300_gpu_archs_list):
|
||||
mi300_models_dict[arch] = []
|
||||
for models in value[MI300]["gpu_archs"][idx]["models"]:
|
||||
gpu_model = models["gpu_model"]
|
||||
|
||||
# 1. Parse compute partition. NOTE: compute partition mode num xcds is available for all mi300 gpu models
|
||||
mi300_num_xcds_dict[gpu_model] = models["partition_mode"][
|
||||
"compute_partition_mode"
|
||||
]["num_xcds"]
|
||||
|
||||
# 2. Parse memory_partition. NOTE: memory partition mode nps is available for all mi300 gpu models
|
||||
mi300_nps_dict[gpu_model] = models["partition_mode"][
|
||||
"memory_partition_mode"
|
||||
]
|
||||
|
||||
# 3. Parse chip id (physical and virtual).
|
||||
if models["chip_ids"]["physical"]:
|
||||
# save chip_id, gpu_model pair if chip id is available
|
||||
# NOTE: chip id is available for all gfx942 machines
|
||||
mi300_chip_id_dict[models["chip_ids"]["physical"]] = models[
|
||||
"gpu_model"
|
||||
]
|
||||
|
||||
if models["chip_ids"]["virtual"]:
|
||||
# save chip_id, gpu_model pair if chip id is available
|
||||
# NOTE: chip id is available for all gfx942 machines
|
||||
mi300_chip_id_dict[models["chip_ids"]["virtual"]] = models[
|
||||
"gpu_model"
|
||||
]
|
||||
|
||||
mi300_models_dict[arch].append(gpu_model)
|
||||
|
||||
gpu_model_dict.update(mi300_models_dict)
|
||||
for series in yaml_data["mi_gpu_spec"]:
|
||||
curr_gpu_series = series["gpu_series"]
|
||||
console_debug("[parse_mi_gpu_spec] Processing series: %s" % curr_gpu_series)
|
||||
for archs in series["gpu_archs"]:
|
||||
curr_gpu_arch = archs["gpu_arch"]
|
||||
gpu_series_dict[curr_gpu_arch] = curr_gpu_series
|
||||
gpu_model_dict[curr_gpu_arch] = []
|
||||
for models in archs["models"]:
|
||||
curr_gpu_model = models["gpu_model"]
|
||||
gpu_model_dict[curr_gpu_arch].append(curr_gpu_model)
|
||||
num_xcds_dict[curr_gpu_model] = (
|
||||
models.get("partition_mode", {})
|
||||
.get("compute_partition_mode", {})
|
||||
.get("num_xcds", {})
|
||||
)
|
||||
if "chip_ids" in models and "physical" in models["chip_ids"]:
|
||||
chip_id_dict[models["chip_ids"]["physical"]] = curr_gpu_model
|
||||
if "chip_ids" in models and "virtual" in models["chip_ids"]:
|
||||
chip_id_dict[models["chip_ids"]["virtual"]] = curr_gpu_model
|
||||
|
||||
|
||||
def get_gpu_series_dict():
|
||||
@@ -164,9 +132,9 @@ def get_gpu_model(gpu_arch_, chip_id_):
|
||||
gpu_arch_lower = gpu_arch_.lower()
|
||||
|
||||
# Handle gfx942 with chip_id mapping
|
||||
if gpu_arch_lower == "gfx942":
|
||||
if chip_id_ and int(chip_id_) in mi300_chip_id_dict:
|
||||
gpu_model = mi300_chip_id_dict.get(int(chip_id_))
|
||||
if gpu_arch_lower not in ("gfx906", "gfx908", "gfx90a"):
|
||||
if chip_id_ and int(chip_id_) in chip_id_dict:
|
||||
gpu_model = chip_id_dict.get(int(chip_id_))
|
||||
else:
|
||||
console_warning(f"No gpu model found for chip id: {chip_id_}")
|
||||
return None
|
||||
@@ -186,8 +154,12 @@ def get_gpu_model(gpu_arch_, chip_id_):
|
||||
return gpu_model.upper()
|
||||
|
||||
|
||||
def get_mi300_num_xcds(gpu_model_, compute_partition_):
|
||||
if not mi300_num_xcds_dict:
|
||||
def get_num_xcds(gpu_model_, compute_partition_):
|
||||
# Only gpu in and above mi 300 series have more than one XCDs
|
||||
if gpu_model_.lower() in ("mi50", "mi60", "mi100", "mi210", "mi250", "mi250x"):
|
||||
return 1
|
||||
|
||||
if not num_xcds_dict:
|
||||
console_error(
|
||||
"mi300_num_xcds_dict not yet populated, did you run parse_mi_gpu_spec()?"
|
||||
)
|
||||
@@ -196,10 +168,10 @@ def get_mi300_num_xcds(gpu_model_, compute_partition_):
|
||||
gpu_model_lower = gpu_model_.lower()
|
||||
partition_lower = compute_partition_.lower()
|
||||
|
||||
if gpu_model_lower not in mi300_num_xcds_dict:
|
||||
if gpu_model_lower not in num_xcds_dict:
|
||||
return None
|
||||
|
||||
model_dict = mi300_num_xcds_dict[gpu_model_lower]
|
||||
model_dict = num_xcds_dict[gpu_model_lower]
|
||||
if partition_lower not in model_dict:
|
||||
console_log(f"Unknown compute partition: {compute_partition_}")
|
||||
return None
|
||||
@@ -214,9 +186,9 @@ def get_mi300_num_xcds(gpu_model_, compute_partition_):
|
||||
return num_xcds
|
||||
|
||||
|
||||
def get_mi300_chip_id_dict():
|
||||
if mi300_chip_id_dict:
|
||||
return mi300_chip_id_dict
|
||||
def get_chip_id_dict():
|
||||
if chip_id_dict:
|
||||
return chip_id_dict
|
||||
else:
|
||||
console_error(
|
||||
"mi300_chip_id_dict not yet populated, did you run parse_mi_gpu_spec()?"
|
||||
|
||||
@@ -9,11 +9,11 @@
|
||||
# MI GPUs
|
||||
# |-- series: the specific MI series; mi50, mi100, mi200, mi300
|
||||
# |-- architecture: currently, only mi300 gpus hold different architectures
|
||||
# |-- models
|
||||
# |-- chip_ids: chip id is specific to the environment the gpu is being used on
|
||||
# |-- partition_mode: currently, only mi300 gpus hold partition mode information
|
||||
# two types: compute partition mode, memory partition mode,
|
||||
# currently only mi300 gpus contains compute partition mode information on number of xcds
|
||||
# |-- gpu model
|
||||
# |-- chip_ids: chip id is specific to the environment the gpu is being used on
|
||||
# |-- partition_mode
|
||||
# | -- compute partition mode
|
||||
# | -- memory partition mode
|
||||
#
|
||||
# --------------------------------------------------------------------------------
|
||||
|
||||
@@ -23,45 +23,31 @@ mi_gpu_spec:
|
||||
- gpu_arch: gfx906
|
||||
models:
|
||||
- gpu_model: mi50
|
||||
partition_mode: null
|
||||
chip_ids:
|
||||
physical: null
|
||||
virtual: null
|
||||
- gpu_model: mi60
|
||||
partition_mode: null
|
||||
chip_ids:
|
||||
physical: null
|
||||
virtual: null
|
||||
|
||||
- gpu_series: mi100
|
||||
gpu_archs:
|
||||
- gpu_arch: gfx908
|
||||
models:
|
||||
- gpu_model: mi100
|
||||
partition_mode: null
|
||||
chip_ids:
|
||||
physical: 29580
|
||||
virtual: null
|
||||
|
||||
- gpu_series: mi200
|
||||
gpu_archs:
|
||||
- gpu_arch: gfx90a
|
||||
models:
|
||||
- gpu_model: mi210
|
||||
partition_mode: null
|
||||
chip_ids:
|
||||
physical: 29711
|
||||
virtual: null
|
||||
- gpu_model: mi250
|
||||
partition_mode: null
|
||||
chip_ids:
|
||||
physical: 29708
|
||||
virtual: null
|
||||
- gpu_model: mi250x
|
||||
partition_mode: null
|
||||
chip_ids:
|
||||
physical: 29704
|
||||
virtual: null
|
||||
- gpu_model: mi250
|
||||
- gpu_model: mi250x
|
||||
|
||||
- gpu_series: mi300
|
||||
gpu_archs:
|
||||
@@ -72,16 +58,10 @@ mi_gpu_spec:
|
||||
compute_partition_mode:
|
||||
num_xcds:
|
||||
spx: 6
|
||||
dpx: null
|
||||
tpx: 2
|
||||
qpx: null
|
||||
cpx: null
|
||||
memory_partition_mode:
|
||||
nps4: [tpx]
|
||||
nps1: [spx, tpx]
|
||||
chip_ids:
|
||||
physical: null
|
||||
virtual: null
|
||||
|
||||
- gpu_arch: gfx941
|
||||
models:
|
||||
@@ -91,15 +71,11 @@ mi_gpu_spec:
|
||||
num_xcds:
|
||||
spx: 8
|
||||
dpx: 4
|
||||
tpx: null
|
||||
qpx: 2
|
||||
cpx: 1
|
||||
memory_partition_mode:
|
||||
nps4: [qpx, cpx]
|
||||
nps1: [spx, qpx, cpx]
|
||||
chip_ids:
|
||||
physical: null
|
||||
virtual: null
|
||||
|
||||
- gpu_arch: gfx942
|
||||
models:
|
||||
@@ -108,10 +84,7 @@ mi_gpu_spec:
|
||||
compute_partition_mode:
|
||||
num_xcds:
|
||||
spx: 6
|
||||
dpx: null
|
||||
tpx: 2
|
||||
qpx: null
|
||||
cpx: null
|
||||
memory_partition_mode:
|
||||
nps4: [tpx]
|
||||
nps1: [spx, tpx]
|
||||
@@ -125,7 +98,6 @@ mi_gpu_spec:
|
||||
num_xcds:
|
||||
spx: 8
|
||||
dpx: 4
|
||||
tpx: null
|
||||
qpx: 2
|
||||
cpx: 1
|
||||
memory_partition_mode:
|
||||
@@ -141,8 +113,6 @@ mi_gpu_spec:
|
||||
num_xcds:
|
||||
spx: 4
|
||||
dpx: 2
|
||||
tpx: null
|
||||
qpx: null
|
||||
cpx: 1
|
||||
memory_partition_mode:
|
||||
nps4: [cpx]
|
||||
@@ -150,3 +120,21 @@ mi_gpu_spec:
|
||||
chip_ids:
|
||||
physical: 29858
|
||||
virtual: 29878
|
||||
|
||||
- gpu_series: mi350
|
||||
gpu_archs:
|
||||
- gpu_arch: gfx950
|
||||
models:
|
||||
- gpu_model: mi350
|
||||
partition_mode:
|
||||
compute_partition_mode:
|
||||
num_xcds:
|
||||
spx: 8
|
||||
dpx: 4
|
||||
qpx: 2
|
||||
cpx: 1
|
||||
memory_partition_mode:
|
||||
nps1: [spx, dpx, qpx, cpx]
|
||||
nps4: [qpx, cpx]
|
||||
chip_ids:
|
||||
physical: 30112
|
||||
|
||||
@@ -86,6 +86,7 @@ build_in_vars = {
|
||||
0) / $max_waves_per_cu) * 8) + MIN(MOD(ROUND(AVG(((4 * SQ_BUSY_CU_CYCLES) \
|
||||
/ $GRBM_GUI_ACTIVE_PER_XCD)), 0), $max_waves_per_cu), 8)), $cu_per_gpu))",
|
||||
"kernelBusyCycles": "ROUND(AVG((((End_Timestamp - Start_Timestamp) / 1000) * $max_sclk)), 0)",
|
||||
"hbmBandwidth": "($max_mclk / 1000 * 32 * $num_hbm_channels)",
|
||||
}
|
||||
|
||||
supported_call = {
|
||||
@@ -700,19 +701,80 @@ def eval_metric(dfs, dfs_type, sys_info, raw_pmc_df, debug):
|
||||
console_error("Hauting execution for warning above.")
|
||||
|
||||
ammolite__se_per_gpu = int(sys_info.se_per_gpu)
|
||||
if np.isnan(ammolite__se_per_gpu) or ammolite__se_per_gpu == 0:
|
||||
console_warning(
|
||||
"se_per_gpu is not available in sysinfo.csv, please provide the correct value using --specs-correction"
|
||||
)
|
||||
ammolite__pipes_per_gpu = int(sys_info.pipes_per_gpu)
|
||||
if np.isnan(ammolite__pipes_per_gpu) or ammolite__pipes_per_gpu == 0:
|
||||
console_warning(
|
||||
"pipes_per_gpu is not available in sysinfo.csv, please provide the correct value using --specs-correction"
|
||||
)
|
||||
ammolite__cu_per_gpu = int(sys_info.cu_per_gpu)
|
||||
if np.isnan(ammolite__cu_per_gpu) or ammolite__cu_per_gpu == 0:
|
||||
console_warning(
|
||||
"cu_per_gpu is not available in sysinfo.csv, please provide the correct value using --specs-correction"
|
||||
)
|
||||
ammolite__simd_per_cu = int(sys_info.simd_per_cu) # not used
|
||||
if np.isnan(ammolite__simd_per_cu) or ammolite__simd_per_cu == 0:
|
||||
console_warning(
|
||||
"simd_per_cu is not available in sysinfo.csv, please provide the correct value using --specs-correction"
|
||||
)
|
||||
ammolite__sqc_per_gpu = int(sys_info.sqc_per_gpu)
|
||||
if np.isnan(ammolite__sqc_per_gpu) or ammolite__sqc_per_gpu == 0:
|
||||
console_warning(
|
||||
"sqc_per_gpu is not available in sysinfo.csv, please provide the correct value using --specs-correction"
|
||||
)
|
||||
ammolite__lds_banks_per_cu = int(sys_info.lds_banks_per_cu)
|
||||
if np.isnan(ammolite__lds_banks_per_cu) or ammolite__lds_banks_per_cu == 0:
|
||||
console_warning(
|
||||
"lds_banks_per_cu is not available in sysinfo.csv, please provide the correct value using --specs-correction"
|
||||
)
|
||||
ammolite__cur_sclk = float(sys_info.cur_sclk) # not used
|
||||
ammolite__mclk = float(sys_info.cur_mclk) # not used
|
||||
if np.isnan(ammolite__cur_sclk) or ammolite__cur_sclk == 0:
|
||||
console_warning(
|
||||
"cur_sclk is not available in sysinfo.csv, please provide the correct value using --specs-correction"
|
||||
)
|
||||
ammolite__cur_mclk = float(sys_info.cur_mclk) # not used
|
||||
if np.isnan(ammolite__cur_mclk) or ammolite__cur_mclk == 0:
|
||||
console_warning(
|
||||
"cur_mclk is not available in sysinfo.csv, please provide the correct value using --specs-correction"
|
||||
)
|
||||
ammolite__max_mclk = float(sys_info.max_mclk)
|
||||
if np.isnan(ammolite__max_mclk) or ammolite__max_mclk == 0:
|
||||
console_warning(
|
||||
"max_mclk is not available in sysinfo.csv, please provide the correct value using --specs-correction"
|
||||
)
|
||||
ammolite__max_sclk = float(sys_info.max_sclk)
|
||||
if np.isnan(ammolite__max_sclk) or ammolite__max_sclk == 0:
|
||||
console_warning(
|
||||
"max_sclk is not available in sysinfo.csv, please provide the correct value using --specs-correction"
|
||||
)
|
||||
ammolite__max_waves_per_cu = int(sys_info.max_waves_per_cu)
|
||||
ammolite__hbm_bw = float(sys_info.hbm_bw)
|
||||
if np.isnan(ammolite__max_waves_per_cu) or ammolite__max_waves_per_cu == 0:
|
||||
console_warning(
|
||||
"max_waver_per_cu is not available in sysinfo.csv, please provide the correct value using --specs-correction"
|
||||
)
|
||||
ammolite__num_hbm_channels = float(sys_info.num_hbm_channels)
|
||||
if np.isnan(ammolite__num_hbm_channels) or ammolite__num_hbm_channels == 0:
|
||||
console_warning(
|
||||
"num_hbm_channels is not available in sysinfo.csv, please provide the correct value using --specs-correction"
|
||||
)
|
||||
ammolite__total_l2_chan = calc_builtin_var("$total_l2_chan", sys_info)
|
||||
if np.isnan(ammolite__total_l2_chan) or ammolite__total_l2_chan == 0:
|
||||
console_warning(
|
||||
"total_l2_chan is not available in sysinfo.csv, please provide the correct value using --specs-correction"
|
||||
)
|
||||
ammolite__num_xcd = int(sys_info.num_xcd)
|
||||
if np.isnan(ammolite__num_xcd) or ammolite__num_xcd == 0:
|
||||
console_warning(
|
||||
"num_xcd is not available in sysinfo.csv, please provide the correct value using --specs-correction"
|
||||
)
|
||||
ammolite__wave_size = int(sys_info.wave_size)
|
||||
if np.isnan(ammolite__wave_size) or ammolite__wave_size == 0:
|
||||
console_warning(
|
||||
"wave_size is not available in sysinfo.csv, please provide the correct value using --specs-correction"
|
||||
)
|
||||
|
||||
# TODO: fix all $normUnit in Unit column or title
|
||||
|
||||
@@ -751,6 +813,7 @@ def eval_metric(dfs, dfs_type, sys_info, raw_pmc_df, debug):
|
||||
ammolite__build_in[key] = None
|
||||
ammolite__numActiveCUs = ammolite__build_in["numActiveCUs"]
|
||||
ammolite__kernelBusyCycles = ammolite__build_in["kernelBusyCycles"]
|
||||
ammolite__hbmBandwidth = ammolite__build_in["hbmBandwidth"]
|
||||
|
||||
# Hmmm... apply + lambda should just work
|
||||
# df['Value'] = df['Value'].apply(lambda s: eval(compile(str(s), '<string>', 'eval')))
|
||||
@@ -821,7 +884,6 @@ def eval_metric(dfs, dfs_type, sys_info, raw_pmc_df, debug):
|
||||
else:
|
||||
console_error("analysis", str(ae))
|
||||
|
||||
# print("eval_metric", id, expr)
|
||||
try:
|
||||
out = eval(compile(row[expr], "<string>", "eval"))
|
||||
|
||||
|
||||
@@ -39,9 +39,9 @@ import pandas as pd
|
||||
|
||||
import config
|
||||
from utils.logger import console_debug, console_error, console_log, console_warning
|
||||
from utils.mi_gpu_spec import get_gpu_series_dict, get_mi300_chip_id_dict
|
||||
from utils.mi_gpu_spec import get_chip_id_dict, get_gpu_series_dict, get_num_xcds
|
||||
from utils.tty import get_table_string
|
||||
from utils.utils import get_version, total_xcds
|
||||
from utils.utils import get_version
|
||||
|
||||
VERSION_LOC = [
|
||||
"version",
|
||||
@@ -72,7 +72,6 @@ def detect_arch(_rocminfo):
|
||||
|
||||
def detect_gpu_chip_id(_rocminfo):
|
||||
gpu_chip_id = None
|
||||
mi300_chip_id_dict = get_mi300_chip_id_dict().keys()
|
||||
|
||||
for idx1, linetext in enumerate(_rocminfo):
|
||||
# NOTE: current supported socs only have numbers in Chip ID
|
||||
@@ -84,8 +83,8 @@ def detect_gpu_chip_id(_rocminfo):
|
||||
if not gpu_chip_id:
|
||||
console_warning("No Chip ID detected: " + str(gpu_chip_id))
|
||||
elif (
|
||||
gpu_chip_id not in mi300_chip_id_dict
|
||||
and int(gpu_chip_id) not in mi300_chip_id_dict
|
||||
gpu_chip_id not in get_chip_id_dict().keys()
|
||||
and int(gpu_chip_id) not in get_chip_id_dict().keys()
|
||||
):
|
||||
console_warning("Unknown Chip ID detected: " + str(gpu_chip_id))
|
||||
return gpu_chip_id
|
||||
@@ -214,7 +213,7 @@ def generate_machine_specs(args, sysinfo: dict = None):
|
||||
specs.total_l2_chan: str = total_l2_banks(
|
||||
specs.gpu_model, int(specs._l2_banks), specs.compute_partition
|
||||
)
|
||||
specs.hbm_bw: str = str(int(specs.max_mclk) / 1000 * 32 * specs.get_hbm_channels())
|
||||
specs.num_hbm_channels: str = str(specs.get_hbm_channels())
|
||||
return specs
|
||||
|
||||
|
||||
@@ -518,15 +517,6 @@ class MachineSpecs:
|
||||
"name": "Pipes per GPU",
|
||||
},
|
||||
)
|
||||
hbm_bw: str = field(
|
||||
default=None,
|
||||
metadata={
|
||||
"doc": "The peak theoretical HBM bandwidth for the accelerators/GPUs in the system. On systems with\n"
|
||||
"configurable partitioning, (e.g., MI300) this is the peak theoretical HBM bandwidth for a partition.",
|
||||
"name": "HBM BW",
|
||||
"unit": "GB/s",
|
||||
},
|
||||
)
|
||||
num_xcd: str = field(
|
||||
default=None,
|
||||
metadata={
|
||||
@@ -536,14 +526,13 @@ class MachineSpecs:
|
||||
"unit": "XCDs",
|
||||
},
|
||||
)
|
||||
num_hbm_channels: str = field(
|
||||
default=None,
|
||||
metadata={"doc": "Number of HBM channels", "name": "HBM channels"},
|
||||
)
|
||||
|
||||
def get_hbm_channels(self):
|
||||
# check MI300 has a valid compute partition
|
||||
mi300a_archs = ["mi300a_a0", "mi300a_a1"]
|
||||
mi300x_archs = ["mi300x_a0", "mi300x_a1"]
|
||||
mi308x_archs = ["mi308x"]
|
||||
|
||||
if self.gpu_model.lower() in mi300a_archs + mi300x_archs + mi308x_archs:
|
||||
if self.memory_partition.lower().startswith("nps"):
|
||||
hbmchannels = 128
|
||||
if self.memory_partition.lower() == "nps2":
|
||||
hbmchannels /= 2
|
||||
@@ -551,10 +540,9 @@ class MachineSpecs:
|
||||
hbmchannels /= 4
|
||||
elif self.memory_partition.lower() == "nps8":
|
||||
hbmchannels /= 8
|
||||
return int(hbmchannels)
|
||||
return hbmchannels
|
||||
else:
|
||||
hbmchannels = int(self.total_l2_chan)
|
||||
return hbmchannels
|
||||
return int(self.total_l2_chan)
|
||||
|
||||
def get_class_members(self):
|
||||
all_populated = True
|
||||
@@ -581,7 +569,7 @@ class MachineSpecs:
|
||||
data[name] = value
|
||||
|
||||
if not all_populated:
|
||||
console_error("Missing specs fields for %s" % self.gpu_arch)
|
||||
console_warning("Missing specs fields for %s" % self.gpu_arch)
|
||||
return pd.DataFrame(data, index=[0])
|
||||
|
||||
def __repr__(self):
|
||||
@@ -682,7 +670,7 @@ def total_sqc(archname, numCUs, numSEs):
|
||||
|
||||
|
||||
def total_l2_banks(archname, L2Banks, compute_partition):
|
||||
xcds = total_xcds(archname, compute_partition)
|
||||
xcds = get_num_xcds(archname, compute_partition)
|
||||
totalL2Banks = L2Banks * xcds
|
||||
return totalL2Banks
|
||||
|
||||
|
||||
@@ -43,16 +43,32 @@ import pandas as pd
|
||||
|
||||
import config
|
||||
from utils.logger import console_debug, console_error, console_log, console_warning
|
||||
from utils.mi_gpu_spec import get_mi300_num_xcds
|
||||
from utils.mi_gpu_spec import get_num_xcds
|
||||
|
||||
rocprof_cmd = ""
|
||||
rocprof_args = ""
|
||||
spi_pipe_counter_regexs = [r"SPI_CS\d+_(.*)", r"SPI_CSQ_P\d+_(.*)"]
|
||||
|
||||
|
||||
def is_tcc_channel_counter(counter):
|
||||
return counter.startswith("TCC") and counter.endswith("]")
|
||||
|
||||
|
||||
def is_spi_pipe_counter(counter):
|
||||
for pattern in spi_pipe_counter_regexs:
|
||||
if re.match(pattern, counter):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def get_base_spi_pipe_counter(counter):
|
||||
for pattern in spi_pipe_counter_regexs:
|
||||
match = re.match(pattern, counter)
|
||||
if match:
|
||||
return match.group(1)
|
||||
return ""
|
||||
|
||||
|
||||
def using_v1():
|
||||
|
||||
return "ROCPROF" not in os.environ.keys() or (
|
||||
@@ -571,12 +587,7 @@ def run_prof(
|
||||
|
||||
# set required env var for mi300
|
||||
new_env = None
|
||||
if (
|
||||
mspec.gpu_model.lower() == "mi300x_a0"
|
||||
or mspec.gpu_model.lower() == "mi300x_a1"
|
||||
or mspec.gpu_model.lower() == "mi300a_a0"
|
||||
or mspec.gpu_model.lower() == "mi300a_a1"
|
||||
):
|
||||
if mspec.gpu_model.lower() not in ("mi50", "mi60", "mi210", "mi250", "mi250x"):
|
||||
new_env = os.environ.copy()
|
||||
new_env["ROCPROFILER_INDIVIDUAL_XCC_MODE"] = "1"
|
||||
|
||||
@@ -661,7 +672,7 @@ def run_prof(
|
||||
if new_env and not using_v3() and not using_v1():
|
||||
# flatten tcc for applicable mi300 input
|
||||
f = path(workload_dir + "/out/pmc_1/results_" + fbase + ".csv")
|
||||
xcds = total_xcds(mspec.gpu_model, mspec.compute_partition)
|
||||
xcds = get_num_xcds(mspec.gpu_model, mspec.compute_partition)
|
||||
df = flatten_tcc_info_across_xcds(f, xcds, int(mspec._l2_banks))
|
||||
df.to_csv(f, index=False)
|
||||
|
||||
@@ -1065,62 +1076,6 @@ def flatten_tcc_info_across_xcds(file, xcds, tcc_channel_per_xcd):
|
||||
return df
|
||||
|
||||
|
||||
def total_xcds(gpu_model, compute_partition):
|
||||
"""
|
||||
Returns the number of xcds for a gpu model and compute_partition pair.
|
||||
"""
|
||||
|
||||
# For mi300 chips, return result from mi_gpu_spec
|
||||
result = get_mi300_num_xcds(gpu_model, compute_partition)
|
||||
if result:
|
||||
return result
|
||||
|
||||
# For other systems, use manual check
|
||||
# check MI300 has a valid compute partition
|
||||
mi300a_model = ["mi300a_a0", "mi300a_a1"]
|
||||
mi300x_model = ["mi300x_a0", "mi300x_a1"]
|
||||
mi308x_model = ["mi308x"]
|
||||
if (
|
||||
gpu_model.lower() in mi300a_model + mi300x_model + mi308x_model
|
||||
and compute_partition == "NA"
|
||||
):
|
||||
console_error("Invalid compute partition found for {}".format(gpu_model))
|
||||
|
||||
if gpu_model.lower() not in mi300a_model + mi300x_model + mi308x_model:
|
||||
return 1
|
||||
# from the whitepaper
|
||||
# https://www.amd.com/content/dam/amd/en/documents/instinct-tech-docs/white-papers/amd-cdna-3-white-paper.pdf
|
||||
if compute_partition.lower() == "spx":
|
||||
if gpu_model.lower() in mi300a_model:
|
||||
return 6
|
||||
if gpu_model.lower() in mi300x_model:
|
||||
return 8
|
||||
if gpu_model.lower() in mi308x_model:
|
||||
return 4
|
||||
if compute_partition.lower() == "tpx":
|
||||
if gpu_model.lower() in mi300a_model:
|
||||
return 2
|
||||
if compute_partition.lower() == "dpx":
|
||||
if gpu_model.lower() in mi300x_model:
|
||||
return 4
|
||||
if gpu_model.lower() in mi308x_model:
|
||||
return 2
|
||||
if compute_partition.lower() == "qpx":
|
||||
if gpu_model.lower() in mi300x_model:
|
||||
return 2
|
||||
if compute_partition.lower() == "cpx":
|
||||
if gpu_model.lower() in mi300x_model:
|
||||
return 1
|
||||
if gpu_model.lower() in mi308x_model:
|
||||
return 1
|
||||
# TODO implement other archs here as needed
|
||||
console_error(
|
||||
"Unknown compute partition / arch found for {} / {}".format(
|
||||
compute_partition, gpu_model
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def get_submodules(package_name):
|
||||
"""List all submodules for a target package"""
|
||||
import importlib
|
||||
|
||||
Ссылка в новой задаче
Block a user