diff --git a/bin/dform.py b/bin/dform.py index 82a81d087b..1ed78d8f6a 100644 --- a/bin/dform.py +++ b/bin/dform.py @@ -46,26 +46,26 @@ def gen_table_bins(db, table, outfile, name_var, dur_ns_var): db.execute('DROP VIEW B') gen_message(outfile) -def gen_api_json_trace(db, table, start_us, outfile): - db.execute('create view B as select "Index", Name as name, pid, tid, (BeginNs/1000 - %d) as ts, (DurationNs/1000) as dur from %s;' % (start_us, table)); +def gen_api_json_trace(db, table, start_ns, outfile): + db.execute('create view B as select "Index", Name as name, pid, tid, ((BeginNs - %d)/1000) as ts, (DurationNs/1000) as dur from %s;' % (start_ns, table)); db.dump_json('B', table, outfile) db.execute('DROP VIEW B') gen_message(outfile) -def gen_ext_json_trace(db, table, start_us, outfile): - db.execute('create view B as select Name as name, pid, tid, (BeginNs/1000 - %d) as ts, ((EndNs - BeginNs)/1000) as dur from %s;' % (start_us, table)); +def gen_ext_json_trace(db, table, start_ns, outfile): + db.execute('create view B as select Name as name, pid, tid, ((BeginNs - %d)/1000) as ts, ((EndNs - BeginNs)/1000) as dur from %s;' % (start_ns, table)); db.dump_json('B', table, outfile) db.execute('DROP VIEW B') gen_message(outfile) -def gen_ops_json_trace(db, table, base_pid, start_us, outfile): - db.execute('create view B as select "Index", Name as name, ("dev-id" + %d) as pid, tid, (BeginNs/1000 - %d) as ts, (DurationNs/1000) as dur from %s;' % (base_pid, start_us, table)); +def gen_ops_json_trace(db, table, base_pid, start_ns, outfile): + db.execute('create view B as select "Index", Name as name, ("dev-id" + %d) as pid, tid, ((BeginNs - %d)/1000) as ts, (DurationNs/1000) as dur from %s;' % (base_pid, start_ns, table)); db.dump_json('B', table, outfile) db.execute('DROP VIEW B') gen_message(outfile) -def gen_kernel_json_trace(db, table, base_pid, start_us, outfile): - db.execute('create view B as select "Index", KernelName as name, ("gpu-id" + %d) as pid, (0) as tid, (BeginNs/1000 - %d) as ts, (DurationNs/1000) as dur from %s;' % (base_pid, start_us, table)); +def gen_kernel_json_trace(db, table, base_pid, start_ns, outfile): + db.execute('create view B as select "Index", KernelName as name, ("gpu-id" + %d) as pid, (0) as tid, ((BeginNs - %d)/1000) as ts, (DurationNs/1000) as dur from %s;' % (base_pid, start_ns, table)); db.dump_json('B', table, outfile) db.execute('DROP VIEW B') gen_message(outfile) diff --git a/bin/sqlitedb.py b/bin/sqlitedb.py index 7aae7c176e..50adb69832 100644 --- a/bin/sqlitedb.py +++ b/bin/sqlitedb.py @@ -151,7 +151,7 @@ class SQLiteDB: fd.write(',{"args":{"name":"%s"},"ph":"M","pid":%s,"name":"process_name","sort_index":%d}\n' %(label, pid, self.section_index)) self.section_index += 1 - def flow_json(self, base_id, from_pid, from_us_list, to_pid, to_us_dict, corr_id_list, start_us, file_name): + def flow_json(self, base_id, from_pid, from_us_list, to_pid, to_us_dict, corr_id_list, file_name): if not re.search(r'\.json$', file_name): raise Exception('wrong output file type: "' + file_name + '"' ) with open(file_name, mode='a') as fd: @@ -160,8 +160,7 @@ class SQLiteDB: corr_id = corr_id_list[ind] if (len(corr_id_list) != 0) else ind if corr_id in to_us_dict: (from_ts, from_tid, to_tid) = from_us_list[ind] - from_ts -= start_us - to_ts = to_us_dict[corr_id] - start_us + to_ts = to_us_dict[corr_id] if from_ts > to_ts: from_ts = to_ts fd.write(',{"ts":%d,"ph":"s","cat":"DataFlow","id":%d,"pid":%d,"tid":%d,"name":"dep"}\n' % (from_ts, dep_id, from_pid, from_tid)) fd.write(',{"ts":%d,"ph":"t","cat":"DataFlow","id":%d,"pid":%d,"tid":%d,"name":"dep"}\n' % (to_ts, dep_id, to_pid, to_tid)) diff --git a/bin/tblextr.py b/bin/tblextr.py index 4ccc92e937..7a198cf11e 100755 --- a/bin/tblextr.py +++ b/bin/tblextr.py @@ -37,7 +37,7 @@ GPU_BASE_PID = 6 NONE_PID = -1 max_gpu_id = -1 -START_US = 0 +START_NS = 0 hsa_activity_found = 0 @@ -164,9 +164,10 @@ def parse_res(infile): var_table[dispatch_number]['CompleteNs'] = m.group(4) ## filling dependenciws - from_ns = m.group(1) - from_us = int(from_ns) / 1000 - to_us = int(m.group(2)) / 1000 + from_ns = int(m.group(1)) + to_ns = int(m.group(2)) + from_us = int((from_ns - START_NS) / 1000) + to_us = int((to_ns - START_NS) / 1000) kern_dep_list.append((from_ns, disp_pid, disp_tid)) @@ -346,6 +347,7 @@ def fill_api_db(table_name, db, indir, api_name, api_pid, dep_pid, dep_list, dep if (hsa_activity_found): copy_raws = db.table_get_raws('COPY') copy_csv = '' copy_index = 0 + op_found = 0 file_name = indir + '/' + api_name + '_api_trace.txt' ptrn_val = re.compile(r'(\d+):(\d+) (\d+):(\d+) ([^\(]+)(\(.*)$') @@ -359,10 +361,6 @@ def fill_api_db(table_name, db, indir, api_name, api_pid, dep_pid, dep_list, dep if not os.path.isfile(file_name): return 0 - dep_tid_list = [] - dep_from_us_list = [] - dep_id_list = [] - # parsing an input trace file and creating a DB table record_id_dict = {} table_handle = db.add_table(table_name, api_table_descr) @@ -410,23 +408,45 @@ def fill_api_db(table_name, db, indir, api_name, api_pid, dep_pid, dep_list, dep # extracting/converting stream id (stream_id, stream_found) = get_field(record_args, 'stream') - if stream_found != 0: + if stream_found == 0: + stream_id = 0 + else: stream_id = get_stream_index(stream_id) (rec_vals[5], found) = set_field(record_args, 'stream', stream_id) if found == 0: fatal('set_field() failed for "stream", args: "' + record_args + '"') - else: stream_id = 0 # extract kernel name string (kernel_str, kernel_found) = get_field(record_args, 'kernel') + if kernel_found == 0: kernel_str = '' + else: op_found = 1 if stream_found != 0 or kernel_found != 0: - ops_patch_data[(corr_id, proc_id)] = (stream_id if stream_found else 0, kernel_str if kernel_found else '') + ops_patch_data[(corr_id, proc_id)] = (stream_id, kernel_str) # dependencies filling if ptrn_ac.match(record_name) or hip_mcopy_ptrn.match(record_name): + op_found = 1 + + # memcopy data + if len(copy_raws) != 0: + copy_data = list(copy_raws[copy_index]) + args_str = rec_vals[5] + args_str = re.sub(r'\(', r'', args_str) + args_str = re.sub(r'\).*$', r'', args_str) + copy_line = str(copy_data[0]) + ', ' + str(copy_data[1]) + ', ' + record_name + ', ' + args_str + copy_csv += str(copy_index) + ', ' + copy_line + '\n' + copy_index += 1 + + if op_found: beg_ns = int(rec_vals[0]) end_ns = int(rec_vals[1]) - from_us = end_ns / 1000 + dur_us = int((end_ns - beg_ns) / 1000) + from_us = int((beg_ns - START_NS) / 1000) + dur_us + + print('beg_end_ns = ' + str(beg_ns) + ':' + str(end_ns)) + print('>> START_NS = ' + str(START_NS)) + print('>> dur_us = ' + str(dur_us)) + print('>> from_us = ' + str(from_us)) if not proc_id in dep_dict: dep_dict[proc_id] = {} dep_proc = dep_dict[proc_id] @@ -439,16 +459,6 @@ def fill_api_db(table_name, db, indir, api_name, api_pid, dep_pid, dep_list, dep dep_str['from'].append((from_us, thrd_id, stream_id)) if expl_id: dep_str['id'].append(corr_id) - # memcopy data - if len(copy_raws) != 0: - copy_data = list(copy_raws[copy_index]) - args_str = rec_vals[5] - args_str = re.sub(r'\(', r'', args_str) - args_str = re.sub(r'\).*$', r'', args_str) - copy_line = str(copy_data[0]) + ', ' + str(copy_data[1]) + ', ' + record_name + ', ' + args_str - copy_csv += str(copy_index) + ', ' + copy_line + '\n' - copy_index += 1 - if False: # patching activity properties: kernel name, stream-id if (corr_id, proc_id) in dep_filtr: @@ -526,11 +536,14 @@ def fill_copy_db(table_name, db, indir): db.insert_entry(table_handle, rec_vals) # filling dependencies + to_ns = int(rec_vals[0]) + to_us = int((to_ns - START_NS) / 1000) + if not proc_id in dep_dict: dep_dict[proc_id] = {} dep_proc = dep_dict[proc_id] if not pid in dep_proc: dep_proc[pid] = { 'pid': HSA_PID, 'from': [], 'to': {}, 'id': [] } dep_str = dep_proc[pid] - dep_str['to'][corr_id] = int(rec_vals[0]) / 1000 + dep_str['to'][corr_id] = to_us else: fatal("async-copy bad record: '" + record + "'") @@ -616,11 +629,14 @@ def fill_ops_db(kernel_table_name, mcopy_table_name, db, indir): filtr[(corr_id, proc_id)] = rec_table_name # filling a dependencies + to_ns = int(rec_vals[0]) + to_us = int((to_ns - START_NS) / 1000) + if not proc_id in dep_dict: dep_dict[proc_id] = {} dep_proc = dep_dict[proc_id] if not pid in dep_proc: dep_proc[pid] = { 'bsp': OPS_PID, 'to': {} } dep_str = dep_proc[pid] - dep_str['to'][corr_id] = int(rec_vals[0]) / 1000 + dep_str['to'][corr_id] = to_us else: fatal("hcc ops bad record: '" + record + "'") @@ -644,10 +660,10 @@ if os.path.isfile(begin_ts_file): with open(begin_ts_file, mode='r') as fd: ind = 0 for line in fd.readlines(): - val = int(line) / 1000 - if ind == 0 or val < START_US: START_US = val + val = int(line) + if ind == 0 or val < START_NS: START_NS = val ind += 1 - print('START timestamp found (' + str(START_US) + 'us)') + print('START timestamp found (' + str(START_NS) + 'ns)') if re.search(r'\.csv$', outfile): csvfile = outfile @@ -718,38 +734,38 @@ else: db.label_json(int(ind) + int(GPU_BASE_PID), "GPU" + str(ind), jsonfile) if ext_trace_found: - dform.gen_ext_json_trace(db, 'rocTX', START_US, jsonfile) + dform.gen_ext_json_trace(db, 'rocTX', START_NS, jsonfile) if len(var_table) != 0: dform.post_process_data(db, 'A', csvfile) dform.gen_table_bins(db, 'A', statfile, 'KernelName', 'DurationNs') if hsa_trace_found and 'BeginNs' in var_list: - dform.gen_kernel_json_trace(db, 'A', GPU_BASE_PID, START_US, jsonfile) + dform.gen_kernel_json_trace(db, 'A', GPU_BASE_PID, START_NS, jsonfile) if hsa_trace_found: dform.post_process_data(db, 'HSA') dform.gen_table_bins(db, 'HSA', hsa_statfile, 'Name', 'DurationNs') - dform.gen_api_json_trace(db, 'HSA', START_US, jsonfile) + dform.gen_api_json_trace(db, 'HSA', START_NS, jsonfile) if copy_trace_found: dform.post_process_data(db, 'COPY') dform.gen_table_bins(db, 'COPY', copy_statfile, 'Name', 'DurationNs') - dform.gen_api_json_trace(db, 'COPY', START_US, jsonfile) + dform.gen_api_json_trace(db, 'COPY', START_NS, jsonfile) if hip_trace_found: dform.post_process_data(db, 'HIP') dform.gen_table_bins(db, 'HIP', hip_statfile, 'Name', 'DurationNs') - dform.gen_api_json_trace(db, 'HIP', START_US, jsonfile) + dform.gen_api_json_trace(db, 'HIP', START_NS, jsonfile) if ops_filtr: dform.post_process_data(db, 'OPS') dform.gen_table_bins(db, 'OPS', ops_statfile, 'Name', 'DurationNs') - dform.gen_ops_json_trace(db, 'OPS', GPU_BASE_PID, START_US, jsonfile) + dform.gen_ops_json_trace(db, 'OPS', GPU_BASE_PID, START_NS, jsonfile) if kfd_trace_found: dform.post_process_data(db, 'KFD') dform.gen_table_bins(db, 'KFD', kfd_statfile, 'Name', 'DurationNs') - dform.gen_api_json_trace(db, 'KFD', START_US, jsonfile) + dform.gen_api_json_trace(db, 'KFD', START_NS, jsonfile) if any_trace_found: dep_id = 0 @@ -771,7 +787,7 @@ else: to_us_dict = dep_str['to'] corr_id_list = dep_str['id'] - db.flow_json(dep_id, from_pid, from_us_list, to_pid, to_us_dict, corr_id_list, START_US, jsonfile) + db.flow_json(dep_id, from_pid, from_us_list, to_pid, to_us_dict, corr_id_list, jsonfile) dep_id += len(from_us_list) if any_trace_found: