AI/rocm-systems
2
0
Fork 0
Code Issues Pull-Requests Actions Pakete Projekte Releases Wiki Aktivität

JSON flow timestamp fix

Quellcode durchsuchen 
Change-Id: I2dacc29250f2d9c5d56bf8ab3ca7f621cb7b663a
Dieser Commit ist enthalten in:
Evgeny
2020-09-21 03:53:03 -04:00
Ursprung a3fafe585d
Commit a87d41c6e4
3 geänderte Dateien mit 61 neuen und 46 gelöschten Zeilen
Patch-Datei herunterladen Vergleichs-Datei herunterladen Alle Dateien ausklappen Alle Dateien einklappen
bin/dform.py
+8 -8
Datei anzeigen
@@ -46,26 +46,26 @@ def gen_table_bins(db, table, outfile, name_var, dur_ns_var):
db.execute('DROP VIEW B')
gen_message(outfile)
def gen_api_json_trace(db, table, start_us, outfile):
db.execute('create view B as select "Index", Name as name, pid, tid, (BeginNs/1000 - %d) as ts, (DurationNs/1000) as dur from %s;' % (start_us, table));
def gen_api_json_trace(db, table, start_ns, outfile):
db.execute('create view B as select "Index", Name as name, pid, tid, ((BeginNs - %d)/1000) as ts, (DurationNs/1000) as dur from %s;' % (start_ns, table));
db.dump_json('B', table, outfile)
db.execute('DROP VIEW B')
gen_message(outfile)
def gen_ext_json_trace(db, table, start_us, outfile):
db.execute('create view B as select Name as name, pid, tid, (BeginNs/1000 - %d) as ts, ((EndNs - BeginNs)/1000) as dur from %s;' % (start_us, table));
def gen_ext_json_trace(db, table, start_ns, outfile):
db.execute('create view B as select Name as name, pid, tid, ((BeginNs - %d)/1000) as ts, ((EndNs - BeginNs)/1000) as dur from %s;' % (start_ns, table));
db.dump_json('B', table, outfile)
db.execute('DROP VIEW B')
gen_message(outfile)
def gen_ops_json_trace(db, table, base_pid, start_us, outfile):
db.execute('create view B as select "Index", Name as name, ("dev-id" + %d) as pid, tid, (BeginNs/1000 - %d) as ts, (DurationNs/1000) as dur from %s;' % (base_pid, start_us, table));
def gen_ops_json_trace(db, table, base_pid, start_ns, outfile):
db.execute('create view B as select "Index", Name as name, ("dev-id" + %d) as pid, tid, ((BeginNs - %d)/1000) as ts, (DurationNs/1000) as dur from %s;' % (base_pid, start_ns, table));
db.dump_json('B', table, outfile)
db.execute('DROP VIEW B')
gen_message(outfile)
def gen_kernel_json_trace(db, table, base_pid, start_us, outfile):
db.execute('create view B as select "Index", KernelName as name, ("gpu-id" + %d) as pid, (0) as tid, (BeginNs/1000 - %d) as ts, (DurationNs/1000) as dur from %s;' % (base_pid, start_us, table));
def gen_kernel_json_trace(db, table, base_pid, start_ns, outfile):
db.execute('create view B as select "Index", KernelName as name, ("gpu-id" + %d) as pid, (0) as tid, ((BeginNs - %d)/1000) as ts, (DurationNs/1000) as dur from %s;' % (base_pid, start_ns, table));
db.dump_json('B', table, outfile)
db.execute('DROP VIEW B')
gen_message(outfile)
bin/sqlitedb.py
+2 -3
Datei anzeigen
@@ -151,7 +151,7 @@ class SQLiteDB:
fd.write(',{"args":{"name":"%s"},"ph":"M","pid":%s,"name":"process_name","sort_index":%d}\n' %(label, pid, self.section_index))
self.section_index += 1
def flow_json(self, base_id, from_pid, from_us_list, to_pid, to_us_dict, corr_id_list, start_us, file_name):
def flow_json(self, base_id, from_pid, from_us_list, to_pid, to_us_dict, corr_id_list, file_name):
if not re.search(r'\.json$', file_name):
raise Exception('wrong output file type: "' + file_name + '"' )
with open(file_name, mode='a') as fd:
@@ -160,8 +160,7 @@ class SQLiteDB:
corr_id = corr_id_list[ind] if (len(corr_id_list) != 0) else ind
if corr_id in to_us_dict:
(from_ts, from_tid, to_tid) = from_us_list[ind]
from_ts -= start_us
to_ts = to_us_dict[corr_id] - start_us
to_ts = to_us_dict[corr_id]
if from_ts > to_ts: from_ts = to_ts
fd.write(',{"ts":%d,"ph":"s","cat":"DataFlow","id":%d,"pid":%d,"tid":%d,"name":"dep"}\n' % (from_ts, dep_id, from_pid, from_tid))
fd.write(',{"ts":%d,"ph":"t","cat":"DataFlow","id":%d,"pid":%d,"tid":%d,"name":"dep"}\n' % (to_ts, dep_id, to_pid, to_tid))
bin/tblextr.py
+51 -35
Datei anzeigen
@@ -37,7 +37,7 @@ GPU_BASE_PID = 6
NONE_PID = -1
max_gpu_id = -1
START_US = 0
START_NS = 0
hsa_activity_found = 0
@@ -164,9 +164,10 @@ def parse_res(infile):
var_table[dispatch_number]['CompleteNs'] = m.group(4)
## filling dependenciws
from_ns = m.group(1)
from_us = int(from_ns) / 1000
to_us = int(m.group(2)) / 1000
from_ns = int(m.group(1))
to_ns = int(m.group(2))
from_us = int((from_ns - START_NS) / 1000)
to_us = int((to_ns - START_NS) / 1000)
kern_dep_list.append((from_ns, disp_pid, disp_tid))
@@ -346,6 +347,7 @@ def fill_api_db(table_name, db, indir, api_name, api_pid, dep_pid, dep_list, dep
if (hsa_activity_found): copy_raws = db.table_get_raws('COPY')
copy_csv = ''
copy_index = 0
op_found = 0
file_name = indir + '/' + api_name + '_api_trace.txt'
ptrn_val = re.compile(r'(\d+):(\d+) (\d+):(\d+) ([^\(]+)(\(.*)$')
@@ -359,10 +361,6 @@ def fill_api_db(table_name, db, indir, api_name, api_pid, dep_pid, dep_list, dep
if not os.path.isfile(file_name): return 0
dep_tid_list = []
dep_from_us_list = []
dep_id_list = []
# parsing an input trace file and creating a DB table
record_id_dict = {}
table_handle = db.add_table(table_name, api_table_descr)
@@ -410,23 +408,45 @@ def fill_api_db(table_name, db, indir, api_name, api_pid, dep_pid, dep_list, dep
# extracting/converting stream id
(stream_id, stream_found) = get_field(record_args, 'stream')
if stream_found != 0:
if stream_found == 0:
stream_id = 0
else:
stream_id = get_stream_index(stream_id)
(rec_vals[5], found) = set_field(record_args, 'stream', stream_id)
if found == 0: fatal('set_field() failed for "stream", args: "' + record_args + '"')
else: stream_id = 0
# extract kernel name string
(kernel_str, kernel_found) = get_field(record_args, 'kernel')
if kernel_found == 0: kernel_str = ''
else: op_found = 1
if stream_found != 0 or kernel_found != 0:
ops_patch_data[(corr_id, proc_id)] = (stream_id if stream_found else 0, kernel_str if kernel_found else '')
ops_patch_data[(corr_id, proc_id)] = (stream_id, kernel_str)
# dependencies filling
if ptrn_ac.match(record_name) or hip_mcopy_ptrn.match(record_name):
op_found = 1
# memcopy data
if len(copy_raws) != 0:
copy_data = list(copy_raws[copy_index])
args_str = rec_vals[5]
args_str = re.sub(r'\(', r'', args_str)
args_str = re.sub(r'\).*$', r'', args_str)
copy_line = str(copy_data[0]) + ', ' + str(copy_data[1]) + ', ' + record_name + ', ' + args_str
copy_csv += str(copy_index) + ', ' + copy_line + '\n'
copy_index += 1
if op_found:
beg_ns = int(rec_vals[0])
end_ns = int(rec_vals[1])
from_us = end_ns / 1000
dur_us = int((end_ns - beg_ns) / 1000)
from_us = int((beg_ns - START_NS) / 1000) + dur_us
print('beg_end_ns = ' + str(beg_ns) + ':' + str(end_ns))
print('>> START_NS = ' + str(START_NS))
print('>> dur_us = ' + str(dur_us))
print('>> from_us = ' + str(from_us))
if not proc_id in dep_dict: dep_dict[proc_id] = {}
dep_proc = dep_dict[proc_id]
@@ -439,16 +459,6 @@ def fill_api_db(table_name, db, indir, api_name, api_pid, dep_pid, dep_list, dep
dep_str['from'].append((from_us, thrd_id, stream_id))
if expl_id: dep_str['id'].append(corr_id)
# memcopy data
if len(copy_raws) != 0:
copy_data = list(copy_raws[copy_index])
args_str = rec_vals[5]
args_str = re.sub(r'\(', r'', args_str)
args_str = re.sub(r'\).*$', r'', args_str)
copy_line = str(copy_data[0]) + ', ' + str(copy_data[1]) + ', ' + record_name + ', ' + args_str
copy_csv += str(copy_index) + ', ' + copy_line + '\n'
copy_index += 1
if False:
# patching activity properties: kernel name, stream-id
if (corr_id, proc_id) in dep_filtr:
@@ -526,11 +536,14 @@ def fill_copy_db(table_name, db, indir):
db.insert_entry(table_handle, rec_vals)
# filling dependencies
to_ns = int(rec_vals[0])
to_us = int((to_ns - START_NS) / 1000)
if not proc_id in dep_dict: dep_dict[proc_id] = {}
dep_proc = dep_dict[proc_id]
if not pid in dep_proc: dep_proc[pid] = { 'pid': HSA_PID, 'from': [], 'to': {}, 'id': [] }
dep_str = dep_proc[pid]
dep_str['to'][corr_id] = int(rec_vals[0]) / 1000
dep_str['to'][corr_id] = to_us
else: fatal("async-copy bad record: '" + record + "'")
@@ -616,11 +629,14 @@ def fill_ops_db(kernel_table_name, mcopy_table_name, db, indir):
filtr[(corr_id, proc_id)] = rec_table_name
# filling a dependencies
to_ns = int(rec_vals[0])
to_us = int((to_ns - START_NS) / 1000)
if not proc_id in dep_dict: dep_dict[proc_id] = {}
dep_proc = dep_dict[proc_id]
if not pid in dep_proc: dep_proc[pid] = { 'bsp': OPS_PID, 'to': {} }
dep_str = dep_proc[pid]
dep_str['to'][corr_id] = int(rec_vals[0]) / 1000
dep_str['to'][corr_id] = to_us
else:
fatal("hcc ops bad record: '" + record + "'")
@@ -644,10 +660,10 @@ if os.path.isfile(begin_ts_file):
with open(begin_ts_file, mode='r') as fd:
ind = 0
for line in fd.readlines():
val = int(line) / 1000
if ind == 0 or val < START_US: START_US = val
val = int(line)
if ind == 0 or val < START_NS: START_NS = val
ind += 1
print('START timestamp found (' + str(START_US) + 'us)')
print('START timestamp found (' + str(START_NS) + 'ns)')
if re.search(r'\.csv$', outfile):
csvfile = outfile
@@ -718,38 +734,38 @@ else:
db.label_json(int(ind) + int(GPU_BASE_PID), "GPU" + str(ind), jsonfile)
if ext_trace_found:
dform.gen_ext_json_trace(db, 'rocTX', START_US, jsonfile)
dform.gen_ext_json_trace(db, 'rocTX', START_NS, jsonfile)
if len(var_table) != 0:
dform.post_process_data(db, 'A', csvfile)
dform.gen_table_bins(db, 'A', statfile, 'KernelName', 'DurationNs')
if hsa_trace_found and 'BeginNs' in var_list:
dform.gen_kernel_json_trace(db, 'A', GPU_BASE_PID, START_US, jsonfile)
dform.gen_kernel_json_trace(db, 'A', GPU_BASE_PID, START_NS, jsonfile)
if hsa_trace_found:
dform.post_process_data(db, 'HSA')
dform.gen_table_bins(db, 'HSA', hsa_statfile, 'Name', 'DurationNs')
dform.gen_api_json_trace(db, 'HSA', START_US, jsonfile)
dform.gen_api_json_trace(db, 'HSA', START_NS, jsonfile)
if copy_trace_found:
dform.post_process_data(db, 'COPY')
dform.gen_table_bins(db, 'COPY', copy_statfile, 'Name', 'DurationNs')
dform.gen_api_json_trace(db, 'COPY', START_US, jsonfile)
dform.gen_api_json_trace(db, 'COPY', START_NS, jsonfile)
if hip_trace_found:
dform.post_process_data(db, 'HIP')
dform.gen_table_bins(db, 'HIP', hip_statfile, 'Name', 'DurationNs')
dform.gen_api_json_trace(db, 'HIP', START_US, jsonfile)
dform.gen_api_json_trace(db, 'HIP', START_NS, jsonfile)
if ops_filtr:
dform.post_process_data(db, 'OPS')
dform.gen_table_bins(db, 'OPS', ops_statfile, 'Name', 'DurationNs')
dform.gen_ops_json_trace(db, 'OPS', GPU_BASE_PID, START_US, jsonfile)
dform.gen_ops_json_trace(db, 'OPS', GPU_BASE_PID, START_NS, jsonfile)
if kfd_trace_found:
dform.post_process_data(db, 'KFD')
dform.gen_table_bins(db, 'KFD', kfd_statfile, 'Name', 'DurationNs')
dform.gen_api_json_trace(db, 'KFD', START_US, jsonfile)
dform.gen_api_json_trace(db, 'KFD', START_NS, jsonfile)
if any_trace_found:
dep_id = 0
@@ -771,7 +787,7 @@ else:
to_us_dict = dep_str['to']
corr_id_list = dep_str['id']
db.flow_json(dep_id, from_pid, from_us_list, to_pid, to_us_dict, corr_id_list, START_US, jsonfile)
db.flow_json(dep_id, from_pid, from_us_list, to_pid, to_us_dict, corr_id_list, jsonfile)
dep_id += len(from_us_list)
if any_trace_found: