From fb7e7f1568b68ad4d3f5a1e4fdc842061bd4385f Mon Sep 17 00:00:00 2001 From: Evgeny Date: Sat, 27 Jun 2020 01:08:22 -0500 Subject: [PATCH] fixing activity correlation for n-process use case Change-Id: I3e52452b586a08fc9cfcb50851ddcc600c7ef910 [ROCm/rocprofiler commit: d013413cb854ef9eb183e1050352a85c57c0a3dc] --- projects/rocprofiler/bin/mem_manager.py | 15 ++- projects/rocprofiler/bin/sqlitedb.py | 15 ++- projects/rocprofiler/bin/tblextr.py | 164 +++++++++++++----------- 3 files changed, 106 insertions(+), 88 deletions(-) diff --git a/projects/rocprofiler/bin/mem_manager.py b/projects/rocprofiler/bin/mem_manager.py index ed9aa67a30..951e34f71b 100755 --- a/projects/rocprofiler/bin/mem_manager.py +++ b/projects/rocprofiler/bin/mem_manager.py @@ -20,7 +20,7 @@ # THE SOFTWARE. ################################################################################ -import os, re +import sys, os, re from sqlitedb import SQLiteDB pinned = ['hipMallocHost', 'hipHostMalloc', 'hipHostAlloc'] @@ -94,11 +94,14 @@ class MemManager: #get type of ptr def get_ptr_type(self, ptr): - ptr_type = '' + ptr_type = 'unknown' found = 0 - for pt in self.allocations.keys(): - (size, event) = self.allocations[pt] - if int(ptr,16) >= int(pt, 16) and int(ptr,16) <= int(pt, 16) + int(size, 16): + for base in self.allocations.keys(): + (size, event) = self.allocations[base] + size = re.sub('\).*$', '', size) + size = '0x' + size + #print("ptr(" + str(ptr) + ") base(" + base + ") size='" + size + "'") + if int(ptr, 16) >= int(base, 16) and int(ptr, 16) < int(base, 16) + int(size, 16): found = 1 break if not found: @@ -107,8 +110,6 @@ class MemManager: ptr_type = 'pinned' elif event in ondevice: ptr_type = 'device' - else: - fatal('internal error: ptr(' + ptr + ') cannot be identified') return ptr_type # add memcpy to map diff --git a/projects/rocprofiler/bin/sqlitedb.py b/projects/rocprofiler/bin/sqlitedb.py index 19a8aac3e1..f24657228e 100644 --- a/projects/rocprofiler/bin/sqlitedb.py +++ b/projects/rocprofiler/bin/sqlitedb.py @@ -151,20 +151,21 @@ class SQLiteDB: fd.write(',{"args":{"name":"%s"},"ph":"M","pid":%s,"name":"process_name","sort_index":%d}\n' %(label, pid, self.section_index)) self.section_index += 1 - def flow_json(self, base_id, from_pid, from_tid, from_us_list, to_pid, to_us_dict, corr_id_list, start_us, file_name): + def flow_json(self, base_id, from_pid, from_us_list, to_pid, to_us_dict, corr_id_list, start_us, file_name): if not re.search(r'\.json$', file_name): raise Exception('wrong output file type: "' + file_name + '"' ) with open(file_name, mode='a') as fd: dep_id = base_id - for ind in range(len(from_tid)): - if (len(corr_id_list) != 0): corr_id = corr_id_list[ind] - else: corr_id = ind + for ind in range(len(from_us_list)): + corr_id = corr_id_list[ind] if (len(corr_id_list) != 0) else ind if corr_id in to_us_dict: - from_ts = from_us_list[ind] - start_us + (from_ts, from_tid) = from_us_list[ind] + from_ts -= start_us + to_tid = 0 to_ts = to_us_dict[corr_id] - start_us if from_ts > to_ts: from_ts = to_ts - fd.write(',{"ts":%d,"ph":"s","cat":"DataFlow","id":%d,"pid":%s,"tid":%s,"name":"dep"}\n' % (from_ts, dep_id, str(from_pid), from_tid[ind])) - fd.write(',{"ts":%d,"ph":"t","cat":"DataFlow","id":%d,"pid":%s,"tid":0,"name":"dep"}\n' % (to_ts, dep_id, str(to_pid))) + fd.write(',{"ts":%d,"ph":"s","cat":"DataFlow","id":%d,"pid":%d,"tid":%d,"name":"dep"}\n' % (from_ts, dep_id, from_pid, from_tid)) + fd.write(',{"ts":%d,"ph":"t","cat":"DataFlow","id":%d,"pid":%d,"tid":%d,"name":"dep"}\n' % (to_ts, dep_id, to_pid, to_tid)) dep_id += 1 def metadata_json(self, jsonfile, sysinfo_file): diff --git a/projects/rocprofiler/bin/tblextr.py b/projects/rocprofiler/bin/tblextr.py index 01817883f9..4eef7ea61a 100755 --- a/projects/rocprofiler/bin/tblextr.py +++ b/projects/rocprofiler/bin/tblextr.py @@ -135,6 +135,7 @@ def parse_res(infile): } gpu_id = 0 + disp_pid = 0 disp_tid = 0 kernel_properties = m.group(2) @@ -148,7 +149,8 @@ def parse_res(infile): if var == 'gpu-id': gpu_id = int(val) if (gpu_id > max_gpu_id): max_gpu_id = gpu_id - if var == 'tid': disp_tid = val + if var == 'pid': disp_pid = int(val) + if var == 'tid': disp_tid = int(val) else: fatal('wrong kernel property "' + prop + '" in "'+ kernel_properties + '"') m = ts_pattern.search(record) if m: @@ -157,20 +159,22 @@ def parse_res(infile): var_table[dispatch_number]['EndNs'] = m.group(3) var_table[dispatch_number]['CompleteNs'] = m.group(4) - gpu_pid = GPU_BASE_PID + int(gpu_id) - if not gpu_pid in dep_dict: dep_dict[gpu_pid] = {} - dep_str = dep_dict[gpu_pid] - if not 'tid' in dep_str: dep_str['tid'] = [] - if not 'from' in dep_str: dep_str['from'] = [] - if not 'to' in dep_str: dep_str['to'] = {} - to_id = len(dep_str['tid']) - from_us = int(m.group(1)) / 1000 + ## filling dependenciws + from_ns = m.group(1) + from_us = int(from_ns) / 1000 to_us = int(m.group(2)) / 1000 + + kern_dep_list.append((from_ns, disp_pid, disp_tid)) + + gpu_pid = GPU_BASE_PID + int(gpu_id) + if not disp_pid in dep_dict: dep_dict[disp_pid] = {} + dep_proc = dep_dict[disp_pid] + if not gpu_pid in dep_proc: dep_proc[gpu_pid] = { 'pid': HSA_PID, 'from': [], 'to': {}, 'id': [] } + dep_str = dep_proc[gpu_pid] + to_id = len(dep_str['from']) + dep_str['from'].append((from_us, disp_tid)) dep_str['to'][to_id] = to_us - dep_str['from'].append(from_us) - dep_str['tid'].append(disp_tid) - dep_str['pid'] = HSA_PID - kern_dep_list.append((disp_tid, m.group(1))) + ## inp.close() ############################################################# @@ -335,7 +339,7 @@ def fill_api_db(table_name, db, indir, api_name, api_pid, dep_pid, dep_list, dep dep_id_list = [] # parsing an input trace file and creating a DB table - record_id = 0 + record_id_dict = {} table_handle = db.add_table(table_name, api_table_descr) with open(file_name, mode='r') as fd: for line in fd.readlines(): @@ -358,20 +362,32 @@ def fill_api_db(table_name, db, indir, api_name, api_pid, dep_pid, dep_list, dep rec_len = len(api_table_descr[0]) - 1 for ind in range(1, rec_len): rec_vals.append(m.group(ind)) - proc_id = rec_vals[2] + proc_id = int(rec_vals[2]) + thrd_id = int(rec_vals[3]) record_name = rec_vals[4] record_args = rec_vals[5] - rec_vals.append(record_id) - corr_id = record_id + + if not proc_id in record_id_dict: record_id_dict[proc_id] = 0 + corr_id = record_id_dict[proc_id] + record_id_dict[proc_id] += 1 + rec_vals.append(corr_id) # dependencies filling if ptrn_ac.search(record_name) or (corr_id, proc_id) in dep_filtr: beg_ns = int(rec_vals[0]) end_ns = int(rec_vals[1]) from_us = (beg_ns / 1000) + ((end_ns - beg_ns) / 1000) - dep_from_us_list.append(from_us) - dep_tid_list.append(int(rec_vals[3])) - dep_id_list.append(corr_id) + + if not proc_id in dep_dict: dep_dict[proc_id] = {} + dep_proc = dep_dict[proc_id] + found = 1 if dep_pid in dep_proc else 0 + if found == 0 and dep_pid == OPS_PID: + dep_proc[dep_pid] = { 'pid': api_pid, 'from': [], 'id': [] } + found = 1 + if found == 1: + dep_str = dep_proc[dep_pid] + dep_str['from'].append((from_us, thrd_id)) + if expl_id: dep_str['id'].append(corr_id) # memcopy data if len(copy_raws) != 0: @@ -391,11 +407,10 @@ def fill_api_db(table_name, db, indir, api_name, api_pid, dep_pid, dep_list, dep if found == 0: fatal('set_field() failed for "stream", args: "' + record_args + '"') # patching activity properties: kernel name, stream-id - corr_id = record_id if (corr_id, proc_id) in dep_filtr: ops_table_name = dep_filtr[(corr_id, proc_id)] - select_expr = '"Index" = ' + str(corr_id) + ' AND "proc-id" = ' + proc_id + select_expr = '"Index" = ' + str(corr_id) + ' AND "proc-id" = ' + str(proc_id) record_args = rec_vals[rec_len - 2] # extract kernel name string @@ -415,26 +430,20 @@ def fill_api_db(table_name, db, indir, api_name, api_pid, dep_pid, dep_list, dep else: activity_record_patching(db, ops_table_name, kernel_found, kernel_str, stream_found, stream_id, select_expr) - rec_vals.append(memory_manager.register_api(rec_vals)) + mcopy_info = memory_manager.register_api(rec_vals) if len(dep_filtr) else '' + rec_vals.append(mcopy_info) rec_vals[2] = api_pid db.insert_entry(table_handle, rec_vals) - record_id += 1 else: fatal(api_name + " bad record: '" + record + "'") # inserting of dispatch events correlated to the dependent dispatches - for (tid, from_ns) in dep_list: - db.insert_entry(table_handle, [from_ns, from_ns, api_pid, tid, 'hsa_dispatch', '', record_id, '']) - record_id += 1 - - # registering dependencies informatino - if dep_pid != NONE_PID: - if not dep_pid in dep_dict: dep_dict[dep_pid] = {} - dep_dict[dep_pid]['pid'] = api_pid - dep_dict[dep_pid]['tid'] = dep_tid_list - dep_dict[dep_pid]['from'] = dep_from_us_list - if expl_id: dep_dict[dep_pid]['id'] = dep_id_list + for (from_ns, proc_id, thrd_id) in dep_list: + if not proc_id in record_id_dict: record_id_dict[proc_id] = 0 + corr_id = record_id_dict[proc_id] + record_id_dict[proc_id] += 1 + db.insert_entry(table_handle, [from_ns, from_ns, api_pid, thrd_id, 'hsa_dispatch', '', corr_id, '']) # generating memcopy CSV if copy_csv != '': @@ -448,19 +457,17 @@ def fill_api_db(table_name, db, indir, api_name, api_pid, dep_pid, dep_list, dep # fill COPY DB copy_table_descr = [ - ['BeginNs', 'EndNs', 'Name', 'pid', 'tid', 'Index'], - {'Index':'INTEGER', 'Name':'TEXT', 'args':'TEXT', 'BeginNs':'INTEGER', 'EndNs':'INTEGER', 'pid':'INTEGER', 'tid':'INTEGER'} + ['BeginNs', 'EndNs', 'Name', 'pid', 'tid', 'Index', 'proc-id'], + {'Index':'INTEGER', 'proc-id':'INTEGER', 'Name':'TEXT', 'args':'TEXT', 'BeginNs':'INTEGER', 'EndNs':'INTEGER', 'pid':'INTEGER', 'tid':'INTEGER'} ] def fill_copy_db(table_name, db, indir): + pid = COPY_PID file_name = indir + '/' + 'async_copy_trace.txt' ptrn_val = re.compile(r'(\d+):(\d+) (.*)$') - ptrn_id = re.compile(r'^async-copy(\d+)$') + ptrn_id = re.compile(r'^async-copy:(\d+):(\d+)$') if not os.path.isfile(file_name): return 0 - if not COPY_PID in dep_dict: dep_dict[COPY_PID] = {} - dep_to_us_dict = {} - table_handle = db.add_table(table_name, copy_table_descr) with open(file_name, mode='r') as fd: for line in fd.readlines(): @@ -471,14 +478,24 @@ def fill_copy_db(table_name, db, indir): for ind in range(1,4): rec_vals.append(m.group(ind)) rec_vals.append(COPY_PID) rec_vals.append(0) - m = ptrn_id.match(rec_vals[2]) - if m: dep_to_us_dict[int(m.group(1))] = int(rec_vals[0]) / 1000 - else: fatal("bad async-copy entry") - rec_vals.append(m.group(1)) - db.insert_entry(table_handle, rec_vals) - else: fatal("async-copy bad record: '" + record + "'") - dep_dict[COPY_PID]['to'] = dep_to_us_dict + m = ptrn_id.match(rec_vals[2]) + if not m: fatal("bad async-copy entry '" + record + "'") + corr_id = int(m.group(1)) + proc_id = int(m.group(2)) + rec_vals.append(corr_id) + rec_vals.append(proc_id) + + db.insert_entry(table_handle, rec_vals) + + # filling dependencies + if not proc_id in dep_dict: dep_dict[proc_id] = {} + dep_proc = dep_dict[proc_id] + if not pid in dep_proc: dep_proc[pid] = { 'pid': HSA_PID, 'from': [], 'to': {}, 'id': [] } + dep_str = dep_proc[pid] + dep_str['to'][corr_id] = int(rec_vals[0]) / 1000 + + else: fatal("async-copy bad record: '" + record + "'") return 1 ############################################################# @@ -500,7 +517,6 @@ def fill_ops_db(kernel_table_name, mcopy_table_name, db, indir): filtr = {} - record_id = 0 kernel_table_handle = db.add_table(kernel_table_name, ops_table_descr) mcopy_table_handle = db.add_table(mcopy_table_name, ops_table_descr) with open(file_name, mode='r') as fd: @@ -516,7 +532,7 @@ def fill_ops_db(kernel_table_name, mcopy_table_name, db, indir): if not m: fatal("bad hcc ops entry '" + record + "'") name = m.group(1) corr_id = int(m.group(2)) - 1 - proc_id = m.group(3) + proc_id = int(m.group(3)) # checking name for memcopy pattern if ptrn_mcopy.search(name): @@ -546,11 +562,12 @@ def fill_ops_db(kernel_table_name, mcopy_table_name, db, indir): # registering a dependency filtr filtr[(corr_id, proc_id)] = rec_table_name - # filling a dependency - if not pid in dep_dict: dep_dict[pid] = {} - if not 'to' in dep_dict[pid]: dep_dict[pid]['to'] = {} - dep_dict[pid]['to'][corr_id] = int(rec_vals[0]) / 1000 - dep_dict[pid]['bsp'] = OPS_PID + # filling a dependencies + if not proc_id in dep_dict: dep_dict[proc_id] = {} + dep_proc = dep_dict[proc_id] + if not pid in dep_proc: dep_proc[pid] = { 'bsp': OPS_PID, 'to': {} } + dep_str = dep_proc[pid] + dep_str['to'][corr_id] = int(rec_vals[0]) / 1000 else: fatal("hcc ops bad record: '" + record + "'") @@ -681,28 +698,27 @@ else: dform.gen_api_json_trace(db, 'KFD', START_US, jsonfile) if any_trace_found: - for (to_pid, dep_str) in dep_dict.items(): - if 'bsp' in dep_str: - bspid = dep_str['bsp'] - base_str = dep_dict[bspid] - for v in ('pid', 'tid', 'from', 'id'): - dep_str[v] = base_str[v] - base_str['inv'] = 1 - dep_id = 0 - for (to_pid, dep_str) in dep_dict.items(): - if 'inv' in dep_str: continue - if not 'to' in dep_str: continue + for (proc_id, dep_proc) in dep_dict.items(): + for (to_pid, dep_str) in dep_proc.items(): + if 'bsp' in dep_str: + bspid = dep_str['bsp'] + base_str = dep_proc[bspid] + for v in ('pid', 'from', 'id'): + dep_str[v] = base_str[v] + base_str['inv'] = 1 - to_us_dict = dep_str['to'] - from_us_list = dep_str['from'] - from_pid = dep_str['pid'] - tid_list = dep_str['tid'] - corr_id_list = [] - if 'id' in dep_str: corr_id_list = dep_str['id'] + for (to_pid, dep_str) in dep_proc.items(): + if 'inv' in dep_str: continue + if not 'to' in dep_str: continue - db.flow_json(dep_id, from_pid, tid_list, from_us_list, to_pid, to_us_dict, corr_id_list, START_US, jsonfile) - dep_id += len(tid_list) + from_pid = dep_str['pid'] + from_us_list = dep_str['from'] + to_us_dict = dep_str['to'] + corr_id_list = dep_str['id'] + + db.flow_json(dep_id, from_pid, from_us_list, to_pid, to_us_dict, corr_id_list, START_US, jsonfile) + dep_id += len(from_us_list) if any_trace_found: db.metadata_json(jsonfile, sysinfo_file)