--stats postprocessing
Change-Id: If9ee0d119c445f44b42b5c97af7b276bca1dc300
Этот коммит содержится в:
Evgeny
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/python
|
||||
from sqlitedb import SQLiteDB
|
||||
|
||||
def post_process_data(db, table_name, start_ns, outfile = ''):
|
||||
# db.add_data_column('A', 'DispDurNs', 'INTEGER', 'BeginNs - DispatchNs')
|
||||
# db.add_data_column('A', 'ComplDurNs', 'INTEGER', 'CompleteNs - EndNs')
|
||||
# db.add_data_column('A', 'TotalDurNs', 'INTEGER', 'CompleteNs - DispatchNs')
|
||||
# db.add_data_column(table_name, 'TimeNs', 'INTEGER', 'BeginNs - %d' % start_ns)
|
||||
db.add_data_column(table_name, 'DurationNs', 'INTEGER', 'EndNs - BeginNs')
|
||||
if outfile != '': db.dump_csv(table_name, outfile)
|
||||
|
||||
def gen_data_bins(db, outfile):
|
||||
db.execute('create view C as select Name, Calls, TotalDurationNs, TotalDurationNs/Calls as AverageNs, TotalDurationNs*100.0/(select sum(TotalDurationNs) from %s) as Percentage from %s order by TotalDurationNs desc;' % ('B', 'B'));
|
||||
db.dump_csv('C', outfile)
|
||||
db.execute('DROP VIEW C')
|
||||
|
||||
def gen_table_bins(db, table, outfile, name_var, dur_ns_var):
|
||||
db.execute('create view B as select (%s) as Name, count(%s) as Calls, sum(%s) as TotalDurationNs from %s group by %s' % (name_var, name_var, dur_ns_var, table, name_var))
|
||||
gen_data_bins(db, outfile)
|
||||
db.execute('DROP VIEW B')
|
||||
|
||||
def gen_api_json_trace(db, table, start_ns, outfile):
|
||||
db.execute('create view B as select "Index", Name as name, pid, tid, (BeginNs/1000 - %d/1000) as ts, (DurationNs/1000) as dur from %s order by ts asc;' % (start_ns, table));
|
||||
db.dump_json('B', table, outfile)
|
||||
db.execute('DROP VIEW B')
|
||||
|
||||
def gen_kernel_json_trace(db, table, base_pid, start_ns, outfile):
|
||||
db.execute('create view B as select "Index", KernelName as name, ("gpu-id" + %d) as pid, (0) as tid, (BeginNs/1000 - %d/1000) as ts, (DurationNs/1000) as dur from %s order by ts asc;' % (base_pid, start_ns, table));
|
||||
db.dump_json('B', table, outfile)
|
||||
db.execute('DROP VIEW B')
|
||||
##############################################################################################
|
||||
+13
-25
@@ -35,8 +35,9 @@ BIN_DIR=$PKG_DIR/bin
|
||||
# PATH to custom HSA and OpenCl runtimes
|
||||
HSA_PATH=$PKG_DIR/lib/hsa
|
||||
|
||||
# HSA runtime trace
|
||||
# runtime API trace
|
||||
HSA_TRACE=0
|
||||
HIP_TRACE=0
|
||||
|
||||
# Generate stats
|
||||
GEN_STATS=0
|
||||
@@ -96,7 +97,7 @@ usage() {
|
||||
echo " --list-derived - to print the list of derived metrics with formulas"
|
||||
echo ""
|
||||
echo " -i <.txt|.xml file> - input file"
|
||||
echo " Input file .txt format, automatically rerun application for every pmc/sqtt line:"
|
||||
echo " Input file .txt format, automatically rerun application for every pmc line:"
|
||||
echo ""
|
||||
echo " # Perf counters group 1"
|
||||
echo " pmc : Wavefronts VALUInsts SALUInsts SFetchInsts FlatVMemInsts LDSInsts FlatLDSInsts GDSInsts VALUUtilization FetchSize"
|
||||
@@ -137,24 +138,21 @@ usage() {
|
||||
echo " --ctx-wait <on|off> - to wait for outstanding contexts on profiler exit [on]"
|
||||
echo " --ctx-limit <max number> - maximum number of outstanding contexts [0 - unlimited]"
|
||||
echo " --heartbeat <rate sec> - to print progress heartbeats [0 - disabled]"
|
||||
echo " --sqtt-size <byte size> - to set SQTT buffer size, aggregate for all SE [0x2000000]"
|
||||
echo " Can be set in KB (1024B) or MB (1048576) units, examples 20K or 20M respectively."
|
||||
echo " --sqtt-local <on|off> - to allocate SQTT buffer in local GPU memory [on]"
|
||||
echo ""
|
||||
echo " --hsa-trace - to trace HSA API"
|
||||
echo " --stats - generating stats and json trace output"
|
||||
echo " --hsa-trace - to trace HSA"
|
||||
echo " --hip-trace - to trace HIP"
|
||||
echo ""
|
||||
echo "Configuration file:"
|
||||
echo " You can set your parameters defaults preferences in the configuration file 'rpl_rc.xml'. The search path sequence: .:${HOME}:<package path>"
|
||||
echo " First the configuration file is looking in the current directory, then in your home, and then in the package directory."
|
||||
echo " Configurable options: 'basenames', 'timestamp', 'ctx-limit', 'heartbeat', 'sqtt-size', 'sqtt-local'."
|
||||
echo " Configurable options: 'basenames', 'timestamp', 'ctx-limit', 'heartbeat'."
|
||||
echo " An example of 'rpl_rc.xml':"
|
||||
echo " <defaults"
|
||||
echo " basenames=off"
|
||||
echo " timestamp=off"
|
||||
echo " ctx-limit=0"
|
||||
echo " heartbeat=0"
|
||||
echo " sqtt-size=0x20M"
|
||||
echo " sqtt-local=on"
|
||||
echo " ></defaults>"
|
||||
echo ""
|
||||
exit 1
|
||||
@@ -193,8 +191,8 @@ run() {
|
||||
fi
|
||||
|
||||
if [ "$HSA_TRACE" = 1 ] ; then
|
||||
export HSA_TOOLS_LIB="libtracer_tool.so libroctracer64.so $HSA_TOOLS_LIB"
|
||||
export ROCTRACER_DOMAIN="hsa"
|
||||
export HSA_TOOLS_LIB="libtracer_tool.so libroctracer64.so $HSA_TOOLS_LIB"
|
||||
fi
|
||||
|
||||
redirection_cmd=""
|
||||
@@ -203,6 +201,7 @@ run() {
|
||||
redirection_cmd="2>&1 | tee $ROCP_OUTPUT_DIR/log.txt"
|
||||
fi
|
||||
|
||||
#unset ROCP_OUTPUT_DIR
|
||||
eval "LD_PRELOAD='$HSA_TOOLS_LIB' $APP_CMD $redirection_cmd"
|
||||
}
|
||||
|
||||
@@ -266,27 +265,16 @@ while [ 1 ] ; do
|
||||
export ROCP_OUTSTANDING_MAX="$2"
|
||||
elif [ "$1" = "--heartbeat" ] ; then
|
||||
export ROCP_OUTSTANDING_MON="$2"
|
||||
elif [ "$1" = "--sqtt-size" ] ; then
|
||||
size_m=`echo "$2" | sed -n "s/^\(.*\)M$/\1/p"`
|
||||
size_k=`echo "$2" | sed -n "s/^\(.*\)K$/\1/p"`
|
||||
if [ -n "$size_m" ] ; then size_b=$((size_m*1024*1024))
|
||||
elif [ -n "$size_k" ] ; then size_b=$((size_k*1024))
|
||||
else size_b=$2
|
||||
fi
|
||||
export ROCP_SQTT_SIZE=$size_b
|
||||
elif [ "$1" = "--sqtt-local" ] ; then
|
||||
if [ "$2" = "on" ] ; then
|
||||
export ROCP_SQTT_LOCAL=1
|
||||
else
|
||||
export ROCP_SQTT_LOCAL=0
|
||||
fi
|
||||
elif [ "$1" = "--hsa-trace" ] ; then
|
||||
ARG_VAL=0
|
||||
HSA_TRACE=1
|
||||
elif [ "$1" = "--hip-trace" ] ; then
|
||||
ARG_VAL=0
|
||||
HIP_TRACE=1
|
||||
elif [ "$1" = "--verbose" ] ; then
|
||||
ARG_VAL=0
|
||||
export ROCP_VERBOSE_MODE=1
|
||||
elif [ "$1" = "-s" ] ; then
|
||||
elif [ "$1" = "--stats" ] ; then
|
||||
ARG_VAL=0
|
||||
GEN_STATS=1
|
||||
else
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
import csv, sqlite3, re
|
||||
|
||||
# SQLite Database class
|
||||
class SQLiteDB:
|
||||
def __init__(self, file_name):
|
||||
self.connection = sqlite3.connect(file_name)
|
||||
self.tables = {}
|
||||
|
||||
def __del__(self):
|
||||
self.connection.close()
|
||||
|
||||
# add DB table
|
||||
def add_table(self, name, descr, extra = ()):
|
||||
(field_list, field_dict) = descr
|
||||
if name in self.tables: raise Exception('table is already added: "' + name + '"')
|
||||
|
||||
# create DB table
|
||||
table_descr = []
|
||||
for field in field_list: table_descr.append('"%s" %s' % (field, field_dict[field]))
|
||||
for item in extra: table_descr.append('"%s" %s' % (item[0], item[1]))
|
||||
stm = 'CREATE TABLE ' + name + ' (%s)' % ', '.join(table_descr)
|
||||
cursor = self.connection.cursor()
|
||||
cursor.execute(stm)
|
||||
self.connection.commit()
|
||||
|
||||
# register table
|
||||
fields_str = ','.join(map(lambda x: '"' + x + '"', field_list))
|
||||
templ_str = ','.join('?' * len(field_list))
|
||||
stm = 'INSERT INTO ' + name + '(' + fields_str + ') VALUES(' + templ_str + ');'
|
||||
self.tables[name] = stm
|
||||
|
||||
return (cursor, stm);
|
||||
|
||||
# add columns to table
|
||||
def add_columns(self, name, columns):
|
||||
cursor = self.connection.cursor()
|
||||
for item in columns:
|
||||
stm = 'ALTER TABLE ' + name + ' ADD COLUMN "%s" %s' % (item[0], item[1])
|
||||
cursor.execute(stm)
|
||||
self.connection.commit()
|
||||
|
||||
# add columns with expression
|
||||
def add_data_column(self, table_name, data_label, data_type, data_expr):
|
||||
cursor = self.connection.cursor()
|
||||
cursor.execute('ALTER TABLE %s ADD COLUMN "%s" %s' % (table_name, data_label, data_type))
|
||||
cursor.execute('UPDATE %s SET %s = (%s);' % (table_name, data_label, data_expr))
|
||||
|
||||
# populate DB table entry
|
||||
def insert_entry(self, table, val_list):
|
||||
(cursor, stm) = table
|
||||
cursor.execute(stm, val_list)
|
||||
|
||||
# populate DB table entry
|
||||
def commit_entry(self, table, val_list):
|
||||
self.insert_entry(table, val_list)
|
||||
self.connection.commit()
|
||||
|
||||
# populate DB table data
|
||||
def insert_table(self, table, reader):
|
||||
for val_list in reader:
|
||||
if not val_list[-1]: val_list.pop()
|
||||
self.insert_entry(table, val_list)
|
||||
self.connection.commit()
|
||||
|
||||
# return table fields list
|
||||
def _get_fields(self, table_name):
|
||||
cursor = self.connection.execute('SELECT * FROM ' + table_name)
|
||||
return list(map(lambda x: '"%s"' % (x[0]), cursor.description))
|
||||
|
||||
# return table raws list
|
||||
def _get_raws(self, table_name):
|
||||
cursor = self.connection.execute('SELECT * FROM ' + table_name)
|
||||
return cursor.fetchall()
|
||||
def _get_raw_by_id(self, table_name, req_id):
|
||||
cursor = self.connection.execute('SELECT * FROM ' + table_name + ' WHERE "Index"=?', (req_id,))
|
||||
raws = cursor.fetchall()
|
||||
if len(raws) != 1:
|
||||
raise Exception('Index is not unique, table "' + table_name + '"')
|
||||
return list(raws[0])
|
||||
|
||||
# dump CSV table
|
||||
def dump_csv(self, table_name, file_name):
|
||||
if not re.search(r'\.csv$', file_name):
|
||||
raise Exception('wrong output file type: "' + file_name + '"' )
|
||||
|
||||
fields = self._get_fields(table_name)
|
||||
with open(file_name, mode='w') as fd:
|
||||
fd.write(','.join(fields) + '\n')
|
||||
for raw in self._get_raws(table_name):
|
||||
fd.write(reduce(lambda a, b: str(a) + ',' + str(b), raw) + '\n')
|
||||
|
||||
# dump JSON trace
|
||||
def open_json(self, file_name):
|
||||
if not re.search(r'\.json$', file_name):
|
||||
raise Exception('wrong output file type: "' + file_name + '"' )
|
||||
with open(file_name, mode='w') as fd:
|
||||
fd.write('{ "traceEvents":[{}\n');
|
||||
|
||||
def close_json(self, file_name):
|
||||
if not re.search(r'\.json$', file_name):
|
||||
raise Exception('wrong output file type: "' + file_name + '"' )
|
||||
with open(file_name, mode='a') as fd:
|
||||
fd.write(']}\n');
|
||||
|
||||
def label_json(self, pid, label, file_name):
|
||||
if not re.search(r'\.json$', file_name):
|
||||
raise Exception('wrong output file type: "' + file_name + '"' )
|
||||
with open(file_name, mode='a') as fd:
|
||||
fd.write(',{"args":{"name":"%s"},"ph":"M","pid":%s,"name":"process_name"}\n' %(label, pid));
|
||||
|
||||
def flow_json(self, base_id, from_pid, from_tid, from_us_list, to_pid, to_us_dict, start_us, file_name):
|
||||
if not re.search(r'\.json$', file_name):
|
||||
raise Exception('wrong output file type: "' + file_name + '"' )
|
||||
with open(file_name, mode='a') as fd:
|
||||
dep_id = base_id
|
||||
for ind in range(len(from_tid)):
|
||||
from_ts = from_us_list[ind] - start_us
|
||||
to_ts = to_us_dict[ind] - start_us
|
||||
fd.write(',{"ts":%d,"ph":"s","cat":"DataFlow","id":%d,"pid":%s,"tid":%d,"name":"dep"}\n' % (from_ts, dep_id, str(from_pid), from_tid[ind]))
|
||||
fd.write(',{"ts":%d,"ph":"t","cat":"DataFlow","id":%d,"pid":%s,"tid":0,"name":"dep"}\n' % (to_ts, dep_id, str(to_pid)))
|
||||
dep_id += 1
|
||||
|
||||
def dump_json(self, table_name, data_name, file_name):
|
||||
if not re.search(r'\.json$', file_name):
|
||||
raise Exception('wrong output file type: "' + file_name + '"' )
|
||||
|
||||
sub_ptrn = re.compile(r'(^"|"$)')
|
||||
name_ptrn = re.compile(r'(name|Name)')
|
||||
|
||||
table_fields = self._get_fields(table_name)
|
||||
table_raws = self._get_raws(table_name)
|
||||
data_fields = self._get_fields(data_name)
|
||||
|
||||
with open(file_name, mode='a') as fd:
|
||||
for raw_index in range(len(table_raws)):
|
||||
values = list(table_raws[raw_index])
|
||||
vals_list = []
|
||||
raw_id = 0;
|
||||
for value_index in range(len(values)):
|
||||
label = table_fields[value_index]
|
||||
value = values[value_index]
|
||||
if name_ptrn.search(label): value = sub_ptrn.sub(r'', value)
|
||||
if label == '"Index"': raw_id = value
|
||||
else: vals_list.append('%s:"%s"' % (label, value))
|
||||
|
||||
data = self._get_raw_by_id(data_name, raw_id)
|
||||
args_list = []
|
||||
for value_index in range(len(data)):
|
||||
label = data_fields[value_index]
|
||||
value = data[value_index]
|
||||
if name_ptrn.search(label): value = sub_ptrn.sub(r'', value)
|
||||
args_list.append('%s:"%s"' % (label, value))
|
||||
|
||||
fd.write(',{"ph":"%s",%s,\n "args":{\n %s\n }\n}\n' % ('X', ','.join(vals_list), ',\n '.join(args_list)))
|
||||
|
||||
# execute query on DB
|
||||
def execute(self, cmd):
|
||||
cursor = self.connection.cursor()
|
||||
cursor.execute(cmd)
|
||||
|
||||
# commit DB
|
||||
def commit(self):
|
||||
self.connection.commit()
|
||||
|
||||
# close DB
|
||||
def close(self):
|
||||
self.connection.close()
|
||||
|
||||
# access DB
|
||||
def get_raws(self, table_name):
|
||||
cur = self.connection.cursor()
|
||||
cur.execute("SELECT * FROM %s" % table_name)
|
||||
return cur.fetchall()
|
||||
|
||||
# return CSV descriptor
|
||||
# list of fields and dictionaly for the fields types
|
||||
def _get_csv_descr(self, table_name, fd):
|
||||
reader = csv.DictReader(fd)
|
||||
field_names = reader.fieldnames
|
||||
if not field_names[-1]: field_names.pop()
|
||||
field_types = {}
|
||||
|
||||
for entry in reader:
|
||||
fields_left = [f for f in field_names if f not in field_types.keys()]
|
||||
# all fields processed
|
||||
if not fields_left: break
|
||||
|
||||
for field in fields_left:
|
||||
data = entry[field]
|
||||
# need data for the field to be processed
|
||||
if len(data) == 0: continue
|
||||
|
||||
if data.isdigit():
|
||||
field_types[field] = "INTEGER"
|
||||
else:
|
||||
field_types[field] = "TEXT"
|
||||
|
||||
if len(fields_left) > 0: raise Exception('types not found for fields: ', fields_left)
|
||||
return (field_names, field_types)
|
||||
|
||||
# add CSV table
|
||||
def add_csv_table(self, table_name, file_name, extra = ()):
|
||||
with open(file_name, mode='r') as fd:
|
||||
# get CSV table descriptor
|
||||
descr = self._get_csv_descr(table_name, fd)
|
||||
# reader to populate the table
|
||||
fd.seek(0)
|
||||
reader = csv.reader(fd)
|
||||
reader.next()
|
||||
table = self.add_table(table_name, descr, extra)
|
||||
self.insert_table(table, reader)
|
||||
|
||||
##############################################################################################
|
||||
+236
-26
@@ -23,6 +23,8 @@
|
||||
################################################################################
|
||||
|
||||
import os, sys, re
|
||||
from sqlitedb import SQLiteDB
|
||||
import dform
|
||||
|
||||
# Parsing results in the format:
|
||||
#dispatch[0], queue_index(0), kernel_name("SimpleConvolution"), time(1048928000311041,1048928006154674,1048928006168274,1048928006170503):
|
||||
@@ -30,8 +32,22 @@ import os, sys, re
|
||||
# SQ_WAVES (4096)
|
||||
# SQ_INSTS_VMEM_RD (36864)
|
||||
|
||||
COPY_PID = 0
|
||||
HSA_PID = 1
|
||||
GPU_BASE_PID = 2
|
||||
max_gpu_id = 0
|
||||
START_NS = 0
|
||||
|
||||
# dependencies dictionary
|
||||
dep_dict = {}
|
||||
kern_dep_list = []
|
||||
|
||||
# global vars
|
||||
var_list = ['Index', 'KernelName', 'DispatchNs', 'BeginNs', 'EndNs', 'CompleteNs']
|
||||
table_descr = [
|
||||
['Index', 'KernelName'],
|
||||
{'Index': 'INTEGER', 'KernelName': 'TEXT'}
|
||||
]
|
||||
var_list = table_descr[0]
|
||||
var_table = {}
|
||||
#############################################################
|
||||
|
||||
@@ -42,10 +58,12 @@ def fatal(msg):
|
||||
|
||||
# parse results method
|
||||
def parse_res(infile):
|
||||
global max_gpu_id
|
||||
if not os.path.isfile(infile): fatal("Error: input file '" + infile + "' not found")
|
||||
inp = open(infile, 'r')
|
||||
|
||||
beg_pattern = re.compile("^dispatch\[(\d*)\],.* kernel-name\(\"([^\"]*)\"\)")
|
||||
beg_pattern = re.compile("^dispatch\[(\d*)\], (.*) kernel-name\(\"([^\"]*)\"\)")
|
||||
prop_pattern = re.compile("([\w-]+)\((\w+)\)");
|
||||
ts_pattern = re.compile(", time\((\d*),(\d*),(\d*),(\d*)\)")
|
||||
var_pattern = re.compile("^\s*([^\s]*)\s+\((\d*)\)")
|
||||
|
||||
@@ -67,56 +85,248 @@ def parse_res(infile):
|
||||
if not dispatch_number in var_table:
|
||||
var_table[dispatch_number] = {
|
||||
'Index': dispatch_number,
|
||||
'KernelName': "\"" + m.group(2) + "\""
|
||||
'KernelName': "\"" + m.group(3) + "\""
|
||||
}
|
||||
|
||||
gpu_id = 0
|
||||
disp_tid = 0
|
||||
|
||||
kernel_properties = m.group(2)
|
||||
for prop in kernel_properties.split(', '):
|
||||
m = prop_pattern.match(prop)
|
||||
if m:
|
||||
var = m.group(1)
|
||||
val = m.group(2)
|
||||
var_table[dispatch_number][var] = val
|
||||
if not var in var_list: var_list.append(var);
|
||||
if var == 'gpu-id':
|
||||
if (val > max_gpu_id): max_gpu_id = val
|
||||
gpu_id = val
|
||||
if var == 'tid': disp_tid = int(val)
|
||||
else: fatal('wrong kernel property "' + prop + '" in "'+ kernel_properties + '"')
|
||||
m = ts_pattern.search(record)
|
||||
if m:
|
||||
var_table[dispatch_number]['DispatchNs'] = m.group(1)
|
||||
var_table[dispatch_number]['BeginNs'] = m.group(2)
|
||||
var_table[dispatch_number]['EndNs'] = m.group(3)
|
||||
var_table[dispatch_number]['CompleteNs'] = m.group(4)
|
||||
else: fatal('bad kernel record "' + record + '"')
|
||||
|
||||
gpu_pid = GPU_BASE_PID + int(gpu_id)
|
||||
if not gpu_pid in dep_dict: dep_dict[gpu_pid] = {}
|
||||
dep_str = dep_dict[gpu_pid]
|
||||
if not 'tid' in dep_str: dep_str['tid'] = []
|
||||
if not 'from' in dep_str: dep_str['from'] = []
|
||||
if not 'to' in dep_str: dep_str['to'] = {}
|
||||
to_id = len(dep_str['tid'])
|
||||
from_us = int(m.group(1)) / 1000
|
||||
to_us = int(m.group(2)) / 1000
|
||||
dep_str['to'][to_id] = to_us
|
||||
dep_str['from'].append(from_us)
|
||||
dep_str['tid'].append(disp_tid)
|
||||
kern_dep_list.append((disp_tid, from_us))
|
||||
|
||||
inp.close()
|
||||
#############################################################
|
||||
|
||||
# print results table method
|
||||
def print_tbl(outfile):
|
||||
# merge results table
|
||||
def merge_table():
|
||||
global var_list
|
||||
if len(var_table) == 0: return 1
|
||||
keys = sorted(var_table.keys(), key=int)
|
||||
|
||||
out = open(outfile, 'w')
|
||||
fields = set(var_table[keys[0]])
|
||||
if 'DispatchNs' in fields:
|
||||
var_list.append('DispatchNs')
|
||||
var_list.append('BeginNs')
|
||||
var_list.append('EndNs')
|
||||
var_list.append('CompleteNs')
|
||||
var_list = [x for x in var_list if x in fields]
|
||||
#############################################################
|
||||
|
||||
keys = var_table.keys()
|
||||
keys.sort(key=int)
|
||||
# dump CSV results
|
||||
def dump_csv(file_name):
|
||||
global var_list
|
||||
keys = sorted(var_table.keys(), key=int)
|
||||
|
||||
entry = var_table[keys[0]]
|
||||
list1 = []
|
||||
for var in var_list:
|
||||
if var in entry:
|
||||
list1.append(var)
|
||||
var_list = list1
|
||||
with open(file_name, mode='w') as fd:
|
||||
fd.write(','.join(var_list) + '\n');
|
||||
for ind in keys:
|
||||
entry = var_table[ind]
|
||||
dispatch_number = entry['Index']
|
||||
if ind != dispatch_number: fatal("Dispatch #" + ind + " index mismatch (" + dispatch_number + ")\n")
|
||||
val_list = [entry[var] for var in var_list]
|
||||
fd.write(','.join(val_list) + '\n');
|
||||
#############################################################
|
||||
|
||||
for var in var_list: out.write(var + ',')
|
||||
out.write("\n")
|
||||
# fill kernels DB
|
||||
def fill_kernel_db(table_name, db):
|
||||
global var_list
|
||||
keys = sorted(var_table.keys(), key=int)
|
||||
|
||||
for var in set(var_list).difference(set(table_descr[1])):
|
||||
table_descr[1][var] = 'INTEGER'
|
||||
table_descr[0] = var_list;
|
||||
|
||||
table_handle = db.add_table(table_name, table_descr)
|
||||
|
||||
for ind in keys:
|
||||
entry = var_table[ind]
|
||||
dispatch_number = entry['Index']
|
||||
if ind != dispatch_number: fatal("Dispatch #" + ind + " index mismatch (" + dispatch_number + ")\n")
|
||||
for var in var_list: out.write(entry[var] + ',')
|
||||
out.write("\n")
|
||||
|
||||
out.close()
|
||||
return 0
|
||||
val_list = [entry[var] for var in var_list]
|
||||
db.insert_entry(table_handle, val_list)
|
||||
#############################################################
|
||||
|
||||
# fill HSA DB
|
||||
hsa_table_descr = [
|
||||
['BeginNs', 'EndNs', 'pid', 'tid', 'Name', 'args', 'Index'],
|
||||
{'Index':'INTEGER', 'Name':'TEXT', 'args':'TEXT', 'BeginNs':'INTEGER', 'EndNs':'INTEGER', 'pid':'INTEGER', 'tid':'INTEGER'}
|
||||
]
|
||||
def fill_hsa_db(table_name, db, indir):
|
||||
global START_NS
|
||||
file_name = indir + '/' + 'hsa_api_trace.txt'
|
||||
ptrn_val = re.compile(r'(\d+):(\d+) (\d+):(\d+) ([^\(]+)(\(.*)$')
|
||||
ptrn_ac = re.compile(r'hsa_amd_memory_async_copy')
|
||||
|
||||
if not COPY_PID in dep_dict: dep_dict[COPY_PID] = {}
|
||||
dep_tid_list = []
|
||||
dep_from_us_list = []
|
||||
|
||||
with open(file_name, mode='r') as fd:
|
||||
line = fd.readline()
|
||||
record = line[:-1]
|
||||
m = ptrn_val.match(record)
|
||||
if m: START_NS = int(m.group(1))
|
||||
START_NS = 0
|
||||
|
||||
record_id = 0
|
||||
table_handle = db.add_table(table_name, hsa_table_descr)
|
||||
with open(file_name, mode='r') as fd:
|
||||
for line in fd.readlines():
|
||||
record = line[:-1]
|
||||
m = ptrn_val.match(record)
|
||||
if m:
|
||||
rec_vals = []
|
||||
for ind in range(1,7):
|
||||
rec_vals.append(m.group(ind))
|
||||
rec_vals[2] = HSA_PID
|
||||
rec_vals.append(record_id)
|
||||
db.insert_entry(table_handle, rec_vals)
|
||||
if ptrn_ac.search(rec_vals[4]):
|
||||
beg_ns = int(rec_vals[0])
|
||||
end_ns = int(rec_vals[1])
|
||||
from_us = (beg_ns / 1000) + ((end_ns - beg_ns) / 1000)
|
||||
dep_from_us_list.append(from_us)
|
||||
dep_tid_list.append(int(rec_vals[3]))
|
||||
record_id += 1
|
||||
else: fatal("hsa bad record")
|
||||
|
||||
for (tid, from_us) in kern_dep_list:
|
||||
db.insert_entry(table_handle, [((from_us - 1) * 1000), from_us * 1000, HSA_PID, tid, 'hsa_dispatch', '', record_id])
|
||||
record_id += 1
|
||||
|
||||
dep_dict[COPY_PID]['tid'] = dep_tid_list
|
||||
dep_dict[COPY_PID]['from'] = dep_from_us_list
|
||||
#############################################################
|
||||
|
||||
# fill COPY DB
|
||||
copy_table_descr = [
|
||||
['BeginNs', 'EndNs', 'Name', 'pid', 'tid', 'Index'],
|
||||
{'Index':'INTEGER', 'Name':'TEXT', 'args':'TEXT', 'BeginNs':'INTEGER', 'EndNs':'INTEGER', 'pid':'INTEGER', 'tid':'INTEGER'}
|
||||
]
|
||||
def fill_copy_db(table_name, db, indir):
|
||||
file_name = indir + '/' + 'async_copy_trace.txt'
|
||||
ptrn_val = re.compile(r'(\d+):(\d+) (.*)$')
|
||||
ptrn_id = re.compile(r'^async-copy(\d+)$')
|
||||
|
||||
if not COPY_PID in dep_dict: dep_dict[COPY_PID] = {}
|
||||
dep_to_us_dict = {}
|
||||
|
||||
table_handle = db.add_table(table_name, copy_table_descr)
|
||||
with open(file_name, mode='r') as fd:
|
||||
for line in fd.readlines():
|
||||
record = line[:-1]
|
||||
m = ptrn_val.match(record)
|
||||
if m:
|
||||
rec_vals = []
|
||||
for ind in range(1,4): rec_vals.append(m.group(ind))
|
||||
rec_vals.append(COPY_PID)
|
||||
rec_vals.append(0)
|
||||
m = ptrn_id.match(rec_vals[2])
|
||||
if m: dep_to_us_dict[int(m.group(1))] = int(rec_vals[0]) / 1000
|
||||
else: fatal("async-copy bad name")
|
||||
rec_vals.append(m.group(1))
|
||||
db.insert_entry(table_handle, rec_vals)
|
||||
else: fatal("async-copy bad record")
|
||||
|
||||
dep_dict[COPY_PID]['to'] = dep_to_us_dict
|
||||
#############################################################
|
||||
# main
|
||||
if (len(sys.argv) < 3): fatal("Usage: " + sys.argv[0] + " <output CSV file> <input result files list>")
|
||||
|
||||
outfile = sys.argv[1]
|
||||
infiles = sys.argv[2:]
|
||||
for f in infiles:
|
||||
parse_res(f)
|
||||
ret = print_tbl(outfile)
|
||||
sys.exit(ret)
|
||||
indir = re.sub(r'\/[^\/]*$', r'', infiles[0])
|
||||
print "indir: '" + indir + "'"
|
||||
|
||||
dbfile = ''
|
||||
csvfile = ''
|
||||
|
||||
if re.search(r'\.csv$', outfile):
|
||||
csvfile = outfile
|
||||
elif re.search(r'\.db$', outfile):
|
||||
dbfile = outfile
|
||||
csvfile = re.sub(r'\.db$', '.csv', outfile)
|
||||
else:
|
||||
fatal("Bad output file '" + outfile + "'")
|
||||
|
||||
for f in infiles: parse_res(f)
|
||||
if len(var_table) == 0: sys.exit(1)
|
||||
merge_table()
|
||||
|
||||
if dbfile == '':
|
||||
dump_csv(csvfile)
|
||||
else:
|
||||
statfile = re.sub(r'\.csv$', '.stats.csv', csvfile)
|
||||
jsonfile = re.sub(r'\.csv$', '.json', csvfile)
|
||||
|
||||
with open(dbfile, mode='w') as fd: fd.truncate()
|
||||
db = SQLiteDB(dbfile)
|
||||
|
||||
fill_hsa_db('HSA', db, indir)
|
||||
fill_copy_db('COPY', db, indir)
|
||||
fill_kernel_db('A', db)
|
||||
|
||||
db.open_json(jsonfile);
|
||||
db.label_json(HSA_PID, "CPU", jsonfile)
|
||||
db.label_json(COPY_PID, "COPY", jsonfile)
|
||||
for ind in range(0, int(max_gpu_id) + 1): db.label_json(int(ind) + int(GPU_BASE_PID), "GPU" + str(ind), jsonfile)
|
||||
|
||||
if 'BeginNs' in var_list:
|
||||
dform.post_process_data(db, 'A', START_NS, csvfile)
|
||||
dform.gen_table_bins(db, 'A', statfile, 'KernelName', 'DurationNs')
|
||||
dform.gen_kernel_json_trace(db, 'A', GPU_BASE_PID, START_NS, jsonfile)
|
||||
else:
|
||||
db.dump_csv('A', csvfile)
|
||||
|
||||
statfile = re.sub(r'stats', r'hsa_stats', statfile)
|
||||
dform.post_process_data(db, 'HSA', START_NS)
|
||||
dform.gen_table_bins(db, 'HSA', statfile, 'Name', 'DurationNs')
|
||||
dform.gen_api_json_trace(db, 'HSA', START_NS, jsonfile)
|
||||
|
||||
dform.post_process_data(db, 'COPY', START_NS)
|
||||
dform.gen_api_json_trace(db, 'COPY', START_NS, jsonfile)
|
||||
|
||||
dep_id = 0
|
||||
for (to_pid, dep_str) in dep_dict.items():
|
||||
tid_list = dep_str['tid']
|
||||
from_us_list = dep_str['from']
|
||||
to_us_dict = dep_str['to']
|
||||
db.flow_json(dep_id, HSA_PID, tid_list, from_us_list, to_pid, to_us_dict, (START_NS / 1000), jsonfile)
|
||||
dep_id += len(tid_list)
|
||||
|
||||
db.close_json(jsonfile);
|
||||
db.close()
|
||||
|
||||
sys.exit(0)
|
||||
#############################################################
|
||||
|
||||
Ссылка в новой задаче
Block a user