AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixing stats without hsa trace

Browse Source 
[ROCm/roctracer commit: a8e799d92b]
This commit is contained in:
Evgeny
2019-01-31 00:56:11 -06:00
parent 2a86bd6e6d
commit 0a102b453e
3 changed files with 61 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/roctracer/bin/rpl_run.sh
+7 -6
View File
@@ -23,18 +23,19 @@
################################################################################
time_stamp=`date +%y%m%d_%H%M%S`
BIN_DIR=`dirname $0`
BIN_DIR=`cd $BIN_DIR; pwd`
BIN_DIR=$(dirname $(realpath $0))
PKG_DIR=$(dirname $BIN_DIR)
ROOT_DIR=$(dirname $PKG_DIR)
RUN_DIR=`pwd`
TMP_DIR="/tmp"
DATA_DIR="rpl_data_${time_stamp}_$$"
PKG_DIR=`echo $BIN_DIR | sed "s/\/bin\/*//"`
BIN_DIR=$PKG_DIR/bin
# PATH to custom HSA and OpenCl runtimes
HSA_PATH=$PKG_DIR/lib/hsa
# roctracer path
if [ -z "$ROCTRACER_PATH" ] ; then ROCTRACER_PATH=$ROOT_DIR/roctracer; fi
# runtime API trace
HSA_TRACE=0
HIP_TRACE=0
@@ -42,7 +43,7 @@ HIP_TRACE=0
# Generate stats
GEN_STATS=0
export LD_LIBRARY_PATH=$PKG_DIR/lib:$PKG_DIR/tool:$PKG_DIR/roctracer/lib:$PKG_DIR/roctracer/tool:$HSA_PATH
export LD_LIBRARY_PATH=$PKG_DIR/lib:$PKG_DIR/tool:$ROCTRACER_PATH/lib:$ROCTRACER_PATH/tool:$HSA_PATH
export PATH=.:$PATH
# enable error logging
projects/roctracer/bin/sqlitedb.py
+20 -9
View File
@@ -1,10 +1,11 @@
import csv, sqlite3, re
import csv, sqlite3, re, sys
# SQLite Database class
class SQLiteDB:
def __init__(self, file_name):
self.connection = sqlite3.connect(file_name)
self.tables = {}
self.json_arg_list_enabled = 0
def __del__(self):
self.connection.close()
@@ -71,6 +72,9 @@ class SQLiteDB:
def _get_raws(self, table_name):
cursor = self.connection.execute('SELECT * FROM ' + table_name)
return cursor.fetchall()
def _get_raws_indexed(self, table_name):
cursor = self.connection.execute('SELECT * FROM ' + table_name + ' order by "Index" asc;')
return cursor.fetchall()
def _get_raw_by_id(self, table_name, req_id):
cursor = self.connection.execute('SELECT * FROM ' + table_name + ' WHERE "Index"=?', (req_id,))
raws = cursor.fetchall()
@@ -128,31 +132,38 @@ class SQLiteDB:
name_ptrn = re.compile(r'(name|Name)')
table_fields = self._get_fields(table_name)
table_raws = self._get_raws(table_name)
table_raws = self._get_raws_indexed(table_name)
data_fields = self._get_fields(data_name)
data_raws = self._get_raws_indexed(data_name)
with open(file_name, mode='a') as fd:
for raw_index in range(len(table_raws)):
values = list(table_raws[raw_index])
table_raws_len = len(table_raws)
for raw_index in range(table_raws_len):
if (raw_index == table_raws_len - 1) or (raw_index % 1000 == 0):
sys.stdout.write( \
"\rdump json " + str(raw_index) + ":" + str(len(table_raws)) + " "*100 \
)
vals_list = []
raw_id = 0;
values = list(table_raws[raw_index])
for value_index in range(len(values)):
label = table_fields[value_index]
value = values[value_index]
if name_ptrn.search(label): value = sub_ptrn.sub(r'', value)
if label == '"Index"': raw_id = value
else: vals_list.append('%s:"%s"' % (label, value))
if label != '"Index"': vals_list.append('%s:"%s"' % (label, value))
data = self._get_raw_by_id(data_name, raw_id)
args_list = []
data = list(data_raws[raw_index])
for value_index in range(len(data)):
label = data_fields[value_index]
value = data[value_index]
if name_ptrn.search(label): value = sub_ptrn.sub(r'', value)
args_list.append('%s:"%s"' % (label, value))
if label != '"Index"': args_list.append('%s:"%s"' % (label, value))
fd.write(',{"ph":"%s",%s,\n "args":{\n %s\n }\n}\n' % ('X', ','.join(vals_list), ',\n '.join(args_list)))
sys.stdout.write('\n')
# execute query on DB
def execute(self, cmd):
cursor = self.connection.cursor()
projects/roctracer/bin/tblextr.py
+34 -27
View File
@@ -110,21 +110,20 @@ def parse_res(infile):
var_table[dispatch_number]['BeginNs'] = m.group(2)
var_table[dispatch_number]['EndNs'] = m.group(3)
var_table[dispatch_number]['CompleteNs'] = m.group(4)
else: fatal('bad kernel record "' + record + '"')
gpu_pid = GPU_BASE_PID + int(gpu_id)
if not gpu_pid in dep_dict: dep_dict[gpu_pid] = {}
dep_str = dep_dict[gpu_pid]
if not 'tid' in dep_str: dep_str['tid'] = []
if not 'from' in dep_str: dep_str['from'] = []
if not 'to' in dep_str: dep_str['to'] = {}
to_id = len(dep_str['tid'])
from_us = int(m.group(1)) / 1000
to_us = int(m.group(2)) / 1000
dep_str['to'][to_id] = to_us
dep_str['from'].append(from_us)
dep_str['tid'].append(disp_tid)
kern_dep_list.append((disp_tid, m.group(1)))
gpu_pid = GPU_BASE_PID + int(gpu_id)
if not gpu_pid in dep_dict: dep_dict[gpu_pid] = {}
dep_str = dep_dict[gpu_pid]
if not 'tid' in dep_str: dep_str['tid'] = []
if not 'from' in dep_str: dep_str['from'] = []
if not 'to' in dep_str: dep_str['to'] = {}
to_id = len(dep_str['tid'])
from_us = int(m.group(1)) / 1000
to_us = int(m.group(2)) / 1000
dep_str['to'][to_id] = to_us
dep_str['from'].append(from_us)
dep_str['tid'].append(disp_tid)
kern_dep_list.append((disp_tid, m.group(1)))
inp.close()
#############################################################
@@ -187,6 +186,8 @@ def fill_hsa_db(table_name, db, indir):
ptrn_val = re.compile(r'(\d+):(\d+) (\d+):(\d+) ([^\(]+)(\(.*)$')
ptrn_ac = re.compile(r'hsa_amd_memory_async_copy')
if not os.path.isfile(file_name): return 0
if not COPY_PID in dep_dict: dep_dict[COPY_PID] = {}
dep_tid_list = []
dep_from_us_list = []
@@ -227,6 +228,8 @@ def fill_hsa_db(table_name, db, indir):
dep_dict[COPY_PID]['tid'] = dep_tid_list
dep_dict[COPY_PID]['from'] = dep_from_us_list
return 1
#############################################################
# fill COPY DB
@@ -267,7 +270,6 @@ if (len(sys.argv) < 3): fatal("Usage: " + sys.argv[0] + " <output CSV file> <inp
outfile = sys.argv[1]
infiles = sys.argv[2:]
indir = re.sub(r'\/[^\/]*$', r'', infiles[0])
print "indir: '" + indir + "'"
dbfile = ''
csvfile = ''
@@ -292,15 +294,19 @@ else:
with open(dbfile, mode='w') as fd: fd.truncate()
db = SQLiteDB(dbfile)
db.open_json(jsonfile);
fill_hsa_db('HSA', db, indir)
fill_copy_db('COPY', db, indir)
hsa_trace_found = fill_hsa_db('HSA', db, indir)
if hsa_trace_found:
fill_copy_db('COPY', db, indir)
fill_kernel_db('A', db)
db.open_json(jsonfile);
db.label_json(HSA_PID, "CPU", jsonfile)
db.label_json(COPY_PID, "COPY", jsonfile)
for ind in range(0, int(max_gpu_id) + 1): db.label_json(int(ind) + int(GPU_BASE_PID), "GPU" + str(ind), jsonfile)
if hsa_trace_found:
db.label_json(HSA_PID, "CPU", jsonfile)
db.label_json(COPY_PID, "COPY", jsonfile)
for ind in range(0, int(max_gpu_id) + 1):
db.label_json(int(ind) + int(GPU_BASE_PID), "GPU" + str(ind), jsonfile)
if 'BeginNs' in var_list:
dform.post_process_data(db, 'A', csvfile)
@@ -309,13 +315,14 @@ else:
else:
db.dump_csv('A', csvfile)
statfile = re.sub(r'stats', r'hsa_stats', statfile)
dform.post_process_data(db, 'HSA')
dform.gen_table_bins(db, 'HSA', statfile, 'Name', 'DurationNs')
dform.gen_api_json_trace(db, 'HSA', START_US, jsonfile)
if hsa_trace_found:
statfile = re.sub(r'stats', r'hsa_stats', statfile)
dform.post_process_data(db, 'HSA')
dform.gen_table_bins(db, 'HSA', statfile, 'Name', 'DurationNs')
dform.gen_api_json_trace(db, 'HSA', START_US, jsonfile)
dform.post_process_data(db, 'COPY')
dform.gen_api_json_trace(db, 'COPY', START_US, jsonfile)
dform.post_process_data(db, 'COPY')
dform.gen_api_json_trace(db, 'COPY', START_US, jsonfile)
dep_id = 0
for (to_pid, dep_str) in dep_dict.items():