From c77a6236e7aeba3d4ad023b8187a23ff60333053 Mon Sep 17 00:00:00 2001 From: Evgeny Date: Tue, 24 Mar 2020 22:01:34 -0500 Subject: [PATCH] adding system info Change-Id: I1f7d23bd7c7e317eed9e177f0c7681226be4bfca --- bin/run_tool.sh | 1 - bin/sqlitedb.py | 17 +++++++- bin/tblextr.py | 29 +++++++++++-- bin/txt2params.py | 108 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 148 insertions(+), 7 deletions(-) create mode 100644 bin/txt2params.py diff --git a/bin/run_tool.sh b/bin/run_tool.sh index 04b1066441..ed1609face 100755 --- a/bin/run_tool.sh +++ b/bin/run_tool.sh @@ -1,5 +1,4 @@ #!/bin/sh - BIN_DIR=`dirname $0` BIN_DIR=`realpath $BIN_DIR` PKG_DIR=${BIN_DIR%/bin} diff --git a/bin/sqlitedb.py b/bin/sqlitedb.py index d52db07eeb..b981fcb47e 100644 --- a/bin/sqlitedb.py +++ b/bin/sqlitedb.py @@ -22,6 +22,7 @@ import csv, sqlite3, re, sys from functools import reduce +from txt2params import gen_params # SQLite Database class class SQLiteDB: @@ -133,13 +134,13 @@ class SQLiteDB: if not re.search(r'\.json$', file_name): raise Exception('wrong output file type: "' + file_name + '"' ) with open(file_name, mode='a') as fd: - fd.write(']}\n'); + fd.write('}') def label_json(self, pid, label, file_name): if not re.search(r'\.json$', file_name): raise Exception('wrong output file type: "' + file_name + '"' ) with open(file_name, mode='a') as fd: - fd.write(',{"args":{"name":"%s %s"},"ph":"M","pid":%s,"name":"process_name"}\n' %(self.section_index, label, pid)); + fd.write(',{"args":{"name":"%s"},"ph":"M","pid":%s,"name":"process_name","sort_index":%d}\n' %(label, pid, self.section_index)) self.section_index += 1 def flow_json(self, base_id, from_pid, from_tid, from_us_list, to_pid, to_us_dict, corr_id_list, start_us, file_name): @@ -158,6 +159,18 @@ class SQLiteDB: fd.write(',{"ts":%d,"ph":"t","cat":"DataFlow","id":%d,"pid":%s,"tid":0,"name":"dep"}\n' % (to_ts, dep_id, str(to_pid))) dep_id += 1 + def metadata_json(self, jsonfile, sysinfo_file): + params = gen_params(sysinfo_file); + with open(jsonfile, mode='a') as fd: + fd.write('],\n') + fd.write('"otherData": {\n') + cnt = 0 + for key in params: + if cnt != 0: fd.write(',\n') + fd.write(' "' + key + '": "' + params[key] + '"') + cnt = cnt + 1 + fd.write('\n }\n') + def dump_json(self, table_name, data_name, file_name): if not re.search(r'\.json$', file_name): raise Exception('wrong output file type: "' + file_name + '"' ) diff --git a/bin/tblextr.py b/bin/tblextr.py index 6e2cd98cc3..542f6be86d 100755 --- a/bin/tblextr.py +++ b/bin/tblextr.py @@ -20,7 +20,7 @@ # THE SOFTWARE. ################################################################################ -import os, sys, re +import os, sys, re, subprocess from sqlitedb import SQLiteDB import dform @@ -63,10 +63,26 @@ def dbglog(msg): fatal("error") ############################################################# +# Dumping sysinfo +sysinfo_begin = 1 +def metadata_gen(sysinfo_file, sysinfo_cmd): + global sysinfo_begin + if not re.search(r'\.txt$', sysinfo_file): + raise Exception('wrong output file type: "' + sysinfo_file + '"' ) + if sysinfo_begin == 1: + sysinfo_begin = 0 + with open(sysinfo_file, mode='w') as fd: fd.write('') + with open(sysinfo_file, mode='a') as fd: fd.write('CMD: ' + sysinfo_cmd + '\n') + status = subprocess.call(sysinfo_cmd + ' >> ' + sysinfo_file, + stderr=subprocess.STDOUT, + shell=True) + if status != 0: + raise Exception('Could not run command: "' + sysinfo_cmd + '"') + # parse results method def parse_res(infile): global max_gpu_id - if not os.path.isfile(infile): return # fatal("Error: input file '" + infile + "' not found") + if not os.path.isfile(infile): return inp = open(infile, 'r') beg_pattern = re.compile("^dispatch\[(\d*)\], (.*) kernel-name\(\"([^\"]*)\"\)") @@ -231,9 +247,9 @@ def fill_ext_db(table_name, db, indir, trace_name, api_pid): continue if cid == 2: - if not pid in range_stack: fatal("ROCTX range begin missed, pid(" + pid + ")"); + if not pid in range_stack: fatal("ROCTX range begin not found, pid(" + pid + ")"); pid_stack = range_stack[pid] - if not tid in pid_stack: fatal("ROCTX range begin missed, tid(" + tid + ")"); + if not tid in pid_stack: fatal("ROCTX range begin not found, tid(" + tid + ")"); rec_stack = pid_stack[tid] rec_vals = rec_stack.pop() rec_vals[1] = tms @@ -495,6 +511,8 @@ else: kfd_statfile = re.sub(r'\.stats\.csv$', r'.kfd_stats.csv', statfile) ops_statfile = statfile copy_statfile = re.sub(r'\.stats\.csv$', r'.copy_stats.csv', statfile) + sysinfo_file = re.sub(r'\.stats\.csv$', r'.sysinfo.txt', statfile) + metadata_gen(sysinfo_file, '/opt/rocm/bin/rocminfo') with open(dbfile, mode='w') as fd: fd.truncate() db = SQLiteDB(dbfile) @@ -565,6 +583,8 @@ else: dform.gen_table_bins(db, 'OPS', ops_statfile, 'Name', 'DurationNs') dform.gen_ops_json_trace(db, 'OPS', GPU_BASE_PID, START_US, jsonfile) + metadata_gen(sysinfo_file, '/opt/rocm/bin/hipcc --version') + if kfd_trace_found: dform.post_process_data(db, 'KFD') dform.gen_table_bins(db, 'KFD', kfd_statfile, 'Name', 'DurationNs') @@ -595,6 +615,7 @@ else: dep_id += len(tid_list) if any_trace_found: + db.metadata_json(jsonfile, sysinfo_file) db.close_json(jsonfile); db.close() diff --git a/bin/txt2params.py b/bin/txt2params.py new file mode 100644 index 0000000000..0958aeea0e --- /dev/null +++ b/bin/txt2params.py @@ -0,0 +1,108 @@ +#!/usr/bin/python + +################################################################################ +# Copyright (c) 2018 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +################################################################################ + +import os, sys, re + +# gen_params() takes a text file like the output of rocminfo cmd and parses it into a map {key,value} +# where key is the param and value is the value of this param +# for example: Threadmodel : "posix" +# it also processes encompasing sections to generate a full param name such as (section names separated by '_'): +# "Agent2_PoolInfo_ISAInfo_ISA1_WorkgroupMaxSizeperDimension_x": "1024(0x400)", +def gen_params(txtfile): + fields = {} + parent_field = '' + nbr_indent = 0 + nbr_indent_prev = 0 + check_for_dims = False + with open(txtfile) as fp: + for line in fp: + me = re.match(r'\*\*\* Done \*\*\*',line) #Marks the end of cmd + if me: + parent_field = '' + nbr_indent = 0 + nbr_indent_prev = 0 + check_for_dims = False + continue + mv = re.match(r'HCC clang version\s+(.*)',line) # outlier: only line with a version number and no ':', special case + if mv: + key = 'HCCclangversion' + val = mv.group(1) + fields[key] = val + continue + # Variable 'check_for_dims' is True for text like this: + # Workgroup Max Size per Dimension: + # x 1024(0x400) + # y 1024(0x400) + # z 1024(0x400) + if check_for_dims == True: + mc = re.match(r'\s*([x|y|z])\s+(.*)',line) + if mc: + key_sav = mc.group(1) + if parent_field != '': + key = parent_field + '_' + mc.group(1) + else: + key = mc.group(1) + val = re.sub(r"\s+", "", mc.group(2)) + fields[key] = val + if key_sav == 'z': + check_for_dims = False + nbr_indent_prev = nbr_indent + mi = re.search(r'^(\s+)\w+', line) + md = re.search(':', line) + if mi: + nbr_indent = len(mi.group(1)) / 2 #indentation cnt + else: + if not md: + tmp = re.sub(r"\s+", "", line) + if tmp.isalnum(): + parent_field = tmp + continue + + if nbr_indent < nbr_indent_prev: + pos = parent_field.rfind('_') + if pos != -1: + parent_field = parent_field[:pos] + + # Process lines such as : + # Segment: GLOBAL; FLAGS: KERNARG, FINE GRAINED + # Size: 131897644(0x7dc992c) KB + for lin in line.split(';'): + lin = re.sub(r"\s+", "", lin) + m = re.match(r'(.*):(.*)', lin) + if m: + key, val = m.group(1), m.group(2) + if parent_field != '': + key = parent_field + '_' + key + if val == '': + mk = re.match(r'.*Dimension',key) + if mk: # expect x,y,z on next 3 lines + check_for_dims = True + parent_field = key + else: + fields[key] = val + else: + if nbr_indent != nbr_indent_prev and not check_for_dims : + parent_field = parent_field + '_' + lin.replace(':','') + + return fields