diff --git a/script/hipap.py b/script/hipap.py index 16ad279987..829a2213c4 100755 --- a/script/hipap.py +++ b/script/hipap.py @@ -1,9 +1,25 @@ #!/usr/bin/python import os, sys, re -HEADER = "hip_prof_str.h" +verbose = 0 +PROF_HEADER = "hip_prof_str.h" +OUTPUT = PROF_HEADER REC_MAX_LEN = 1024 +# Fatal error termination +inp_file = 'none' +line_num = -1 +def fatal(msg): + if line_num != -1: + print >>sys.stderr, "Error: " + msg + ", file '" + inp_file + "', line (" + str(line_num) + ")" + else: + print >>sys.stderr, "Error: " + msg + sys.exit(1) + +# Verbose message +def message(msg): + if verbose: print >>sys.stdout, msg + ############################################################# # Normalizing API arguments def filtr_api_args(args_str): @@ -35,8 +51,7 @@ def list_api_args(args_str): arg_name = m.group(2) args_list.append((arg_type, arg_name)) else: - print "Bad args: args_str: '" + args_str + "' arg_pair: '" + arg_pair + "'" - sys.exit(1) + fatal("bad args: args_str: '" + args_str + "' arg_pair: '" + arg_pair + "'") return args_list; # Creating arguments string "type0, type1, ..." @@ -47,7 +62,7 @@ def filtr_api_types(args_str): types_str += arg_tuple[0] + ', ' return types_str -# Creating options list [name0, name1, ...] +# Creating options list [opt0, opt1, ...] def filtr_api_opts(args_str): args_list = list_api_args(args_str) opts_list = [] @@ -57,7 +72,11 @@ def filtr_api_opts(args_str): ############################################################# # Parsing API header # hipError_t hipSetupArgument(const void* arg, size_t size, size_t offset); -def parse_api(inp_file, out): +def parse_api(inp_file_p, out): + global inp_file + global line_num + inp_file = inp_file_p + beg_pattern = re.compile("^(hipError_t|const char\s*\*)\s+[^\(]+\("); api_pattern = re.compile("^(hipError_t|const char\s*\*)\s+([^\(]+)\(([^\)]*)\)"); end_pattern = re.compile("Texture"); @@ -78,8 +97,7 @@ def parse_api(inp_file, out): line_num += 1 if len(record) > REC_MAX_LEN: - print "Error: bad record \"" + record + "\"\nfile '" + inp_file + ", line (" + str(line_num) + ")" - break; + fatal("bad record \"" + record + "\"") if beg_pattern.match(record) and (hidden == 0) and (nms_level == 0): found = 1 @@ -98,20 +116,24 @@ def parse_api(inp_file, out): if nms_open_pattern.match(line): nms_level += 1 if (nms_level > 0) and nms_close_pattern.match(line): nms_level -= 1 if nms_level < 0: - print "Error: nms level < 0" - sys.exit(1) + fatal("nms level < 0") record = "" inp.close() + line_num = -1 ############################################################# # Patching API implementation # hipError_t hipSetupArgument(const void* arg, size_t size, size_t offset) { # HIP_INIT_CB(hipSetupArgument, arg, size, offset); # inp_file - input implementation source file # api_map - input public API map [] => -# out - output map [] => -def patch_content(inp_file, api_map, out): +# out - output map [] => [opt0, opt1, ...] +def parse_content(inp_file_p, api_map, out): + global inp_file + global line_num + inp_file = inp_file_p + # API definition begin pattern beg_pattern = re.compile("^(hipError_t|const char\s*\*)\s+[^\(]+\("); # API definition complete pattern @@ -146,7 +168,7 @@ def patch_content(inp_file, api_map, out): line_num += 1 if len(record) > REC_MAX_LEN: - print "Error: bad record \"" + record + "\"\nfile '" + inp_file + ", line (" + str(line_num) + ")" + fatal("bad record \"" + record + "\"") break; # Looking for API begin @@ -173,27 +195,18 @@ def patch_content(inp_file, api_map, out): api_types = filtr_api_types(api_args) # Normalizing etalon arguments eta_types = filtr_api_types(eta_args) - # Comparing API and etalon arguments - # Normalizing types if not matching - api_types_n = api_types - eta_types_n = eta_types - #if api_types != eta_types: - # api_types_n = norm_api_types(api_types) - # eta_types_n = norm_api_types(eta_types) - # Comparing API and etalon arguments - if api_types_n == eta_types_n: + if api_types == eta_types: # API is already found if api_name in out: - print "Error: API redefined \"" + api_name + "\", record \"" + record + "\"\nfile '" + inp_file + "', line (" + str(line_num) + ")" - sys.exit(1) + fatal("API redefined \"" + api_name + "\", record \"" + record + "\"") # Set valid public API found flag api_valid = 1 # Set output API map with API arguments list out[api_name] = filtr_api_opts(api_args) else: # Warning about mismatched API, possible non public overloaded version - api_diff = '\t\t' + inp_file + " line(" + str(line_num) + ")\n\t\tapi: " + api_types_n + "\n\t\teta: " + eta_types_n - print "\t" + api_name + ':\n' + api_diff + '\n' + api_diff = '\t\t' + inp_file + " line(" + str(line_num) + ")\n\t\tapi: " + api_types + "\n\t\teta: " + eta_types + message("\t" + api_name + ':\n' + api_diff + '\n') # API found action if found == 2: @@ -203,7 +216,7 @@ def patch_content(inp_file, api_map, out): found = 0 if api_valid == 1: api_valid = 0 - print (api_name); + message("\t" + api_name) else: # Registering dummy API for non public API if the name in INIT is not NONE init_name = m.group(1) @@ -211,14 +224,12 @@ def patch_content(inp_file, api_map, out): if init_name != 'NONE': # Check if init name matching API name if init_name != api_name: - print "Init name mismatch:", init_name, "<>", api_name - sys.exit(1) + fatal("init name mismatch: '" + init_name + "' <> '" + api_name + "'") # If init name is not in public API map then it is private API # else it was not identified and will be checked on finish if not init_name in api_map: if init_name in out: - print "Error: API reinit \"" + api_name + "\", record \"" + record + "\"\nfile '" + inp_file + "', line (" + str(line_num) + ")" - sys.exit(1) + fatal("API reinit \"" + api_name + "\", record \"" + record + "\"") out[init_name] = [] elif re.search('}', line): found = 0 @@ -230,48 +241,177 @@ def patch_content(inp_file, api_map, out): del api_map[api_name] out['.' + api_name] = 1 else: - print "Error: API is not in out \"" + api_name + "\", record \"" + record + "\"\nfile '" + inp_file + "', line (" + str(line_num) + ")" - sys.exit(1) + fatal("API is not in out \"" + api_name + "\", record \"" + record + "\"") if found != 1: record = "" content += line inp.close() + line_num = -1 if len(out) != 0: return content else: return '' -# srcs path walk -def patch_src(api_map, src_path, src_patt, out): +# src path walk +def parse_src(api_map, src_path, src_patt, out): pattern = re.compile(src_patt) src_path = re.sub(r'\s', '', src_path) for src_dir in src_path.split(':'): - print "Parsing " + src_dir + " for '" + src_patt + "'" + message("Parsing " + src_dir + " for '" + src_patt + "'") for root, dirs, files in os.walk(src_dir): for fnm in files: if pattern.search(fnm): file = root + '/' + fnm - print "\t" + file - content = patch_content(file, api_map, out); + message(file) + content = parse_content(file, api_map, out); if content != '': f = open(file, 'w') f.write(content) f.close() ############################################################# +# Generating profiling primitives header +# api_map - public API map [] => [(type, name), ...] +# opts_map - opts map [] => [opt0, opt1, ...] +def generate_prof_header(f, api_map, opts_map): + # Private API list + priv_lst = [] + + f.write('// automatically generated sources\n') + f.write('#ifndef _HIP_PROF_STR_H\n'); + f.write('#define _HIP_PROF_STR_H\n'); + f.write('#include \n'); + f.write('#include \n'); + + # Generating dummy macro for non-public API + f.write('\n// Dummy API primitives\n') + f.write('#define INIT_NONE_CB_ARGS_DATA(cb_data) {};\n') + for name in opts_map: + if not name in api_map: + opts_lst = opts_map[name] + if len(opts_lst) != 0: + fatal("bad dummy API \"" + name + "\", args: " + str(opts_lst)) + f.write('#define INIT_'+ name + '_CB_ARGS_DATA(cb_data) {};\n') + priv_lst.append(name) + + for name in priv_lst: + message("Private: " + name) + + # Generating the callbacks ID enumaration + f.write('\n// HIP API callbacks ID enumaration\n') + f.write('enum hip_api_id_t {\n') + cb_id = 0 + for name in api_map.keys(): + f.write(' HIP_API_ID_' + name + ' = ' + str(cb_id) + ',\n') + cb_id += 1 + f.write(' HIP_API_ID_NUMBER = ' + str(cb_id) + ',\n') + f.write(' HIP_API_ID_ANY = ' + str(cb_id + 1) + ',\n') + f.write('\n') + f.write(' HIP_API_ID_NONE = HIP_API_ID_NUMBER,\n') + for name in priv_lst: + f.write(' HIP_API_ID_' + name + ' = HIP_API_ID_NUMBER,\n') + f.write('};\n') + + # Generating the callbacks ID enumaration + f.write('\n// Return HIP API string\n') + f.write('static const char* hip_api_name(const uint32_t& id) {\n') + f.write(' switch(id) {\n') + for name in api_map.keys(): + f.write(' case HIP_API_ID_' + name + ': return "' + name + '";\n') + f.write(' };\n') + f.write(' return "unknown";\n') + f.write('};\n') + + # Generating the callbacks data structure + f.write('\n// HIP API callbacks data structure\n') + f.write( + 'struct hip_api_data_t {\n' + + ' uint64_t correlation_id;\n' + + ' uint32_t phase;\n' + + ' union {\n' + ) + for name, args in api_map.items(): + if len(args) != 0: + f.write(' struct {\n') + for arg_tuple in args: + f.write(' ' + arg_tuple[0] + ' ' + arg_tuple[1] + ';\n') + f.write(' } ' + name + ';\n') + f.write( + ' } args;\n' + + '};\n' + ) + + # Generating the callbacks args data filling macros + f.write('\n// HIP API callbacks args data filling macros\n') + for name, args in api_map.items(): + f.write('// ' + name + str(args) + '\n') + f.write('#define INIT_' + name + '_CB_ARGS_DATA(cb_data) { \\\n') + if name in opts_map: + opts_list = opts_map[name] + if len(args) != len(opts_list): + fatal("\"" + name + "\" API args and opts mismatch, args: " + str(args) + ", opts: " + str(opts_list)) + # API args iterating: + # type is args[][0] + # name is args[][1] + for ind in range(0, len(args)): + arg_tuple = args[ind] + fld_name = arg_tuple[1] + arg_name = opts_list[ind] + f.write(' cb_data.args.' + name + '.' + fld_name + ' = ' + arg_name + '; \\\n') + f.write('};\n') + f.write('#define INIT_CB_ARGS_DATA(cb_id, cb_data) INIT_##cb_id##_CB_ARGS_DATA(cb_data)\n') + + # Generating the method for the API string, name and parameters + f.write('\n') + f.write('#if 0\n') + f.write('// HIP API string method, method name and parameters\n') + f.write('const char* hipApiString(hip_api_id_t id, const hip_api_data_t* data) {\n') + f.write(' std::ostringstream oss;\n') + f.write(' switch (id) {\n') + for name, args in api_map.items(): + f.write(' case HIP_API_ID_' + name + ':\n') + f.write(' oss << "' + name + '("') + for ind in range(0, len(args)): + arg_tuple = args[ind] + arg_name = arg_tuple[1] + if ind != 0: f.write(' << ","') + f.write('\n << " ' + arg_name + '=" << data->args.' + name + '.' + arg_name) + f.write('\n << ")";\n') + f.write(' break;\n') + f.write(' default: oss << "unknown";\n') + f.write(' };\n') + f.write(' return strdup(oss.str().c_str());\n') + f.write('};\n') + f.write('#endif\n') + + f.write('#endif // _HIP_PROF_STR_H\n'); + +############################################################# # main # Usage -if (len(sys.argv) < 2): - print >>sys.stderr, "Usage:", sys.argv[0], " [patched srcs path]" - print >>sys.stderr, " $ hipap.py hip/include/hip/hcc_detail/hip_runtime_api.h hip/src" - sys.exit(1) +if (len(sys.argv) > 1) and (sys.argv[1] == '-v'): + verbose = 1 + sys.argv.pop(1) + +if (len(sys.argv) < 3): + fatal ("Usage: " + sys.argv[0] + " [-v] \n" + + " -v - verbose messages\n" + + " example:\n" + + " $ hipap.py hip/include/hip/hcc_detail/hip_runtime_api.h hip/src") # API header file given as an argument api_hfile = sys.argv[1] if not os.path.isfile(api_hfile): - print >>sys.stderr, "Error: input file '" + api_hfile + "' not found" - sys.exit(1) + fatal("input file '" + api_hfile + "' not found") + +# Srcs directory given as an argument +src_pat = "\.cpp$" +src_dir = sys.argv[2] +if not os.path.isdir(src_dir): + fatal("src directory " + src_dir + "' not found") + +if len(sys.argv) > 3: OUTPUT = sys.argv[3] # API declaration map api_map = { @@ -279,148 +419,36 @@ api_map = { } # API options map opts_map = {} -# Private API list -priv_lst = [] # Parsing API header parse_api(api_hfile, api_map) -# Patching API implementation sources -# Sources path is given as an argument -if len(sys.argv) == 3: - src_path = sys.argv[2] - src_patt = "\.cpp$" - patch_src(api_map, src_path, src_patt, opts_map) +# Parsing sources +parse_src(api_map, src_dir, src_pat, opts_map) # Checking for non-conformant APIs for name in opts_map.keys(): m = re.match(r'\.(\S*)', name) if m: - print "Init missing: " + m.group(1) + message("Init missing: " + m.group(1)) del opts_map[name] # Converting api map to map of lists -# Printing not found APIs +# Checking for not found APIs not_found = 0 if len(opts_map) != 0: for name in api_map.keys(): args_str = api_map[name]; api_map[name] = list_api_args(args_str) if not name in opts_map: - print "Not found: " + name + fatal("not found: " + name) not_found += 1 if not_found != 0: - print "Error:", not_found, "API calls not found" - sys.exit(1) -############################################################# + fatal(not_found + " API calls not found") -f = open(HEADER, 'w') -f.write('// automatically generated sources\n') -f.write('#ifndef _HIP_PROF_STR_H\n'); -f.write('#define _HIP_PROF_STR_H\n'); -f.write('#include \n'); -f.write('#include \n'); +# Generating output header file +with open(OUTPUT, 'w') as f: + generate_prof_header(f, api_map, opts_map) -# Generating dummy macro for non-public API -f.write('\n// Dummy API primitives\n') -f.write('#define INIT_NONE_CB_ARGS_DATA(cb_data) {};\n') -for name in opts_map: - if not name in api_map: - opts_lst = opts_map[name] - if len(opts_lst) != 0: - print ("Error: bad dummy API \"" + name + "\", args: ", opts_lst) - sys.exit(1) - f.write('#define INIT_'+ name + '_CB_ARGS_DATA(cb_data) {};\n') - priv_lst.append(name) - -for name in priv_lst: - print "Private: ", name - -# Generating the callbacks ID enumaration -f.write('\n// HIP API callbacks ID enumaration\n') -f.write('enum hip_api_id_t {\n') -cb_id = 0 -for name in api_map.keys(): - f.write(' HIP_API_ID_' + name + ' = ' + str(cb_id) + ',\n') - cb_id += 1 -f.write(' HIP_API_ID_NUMBER = ' + str(cb_id) + ',\n') -f.write(' HIP_API_ID_ANY = ' + str(cb_id + 1) + ',\n') -f.write('\n') -f.write(' HIP_API_ID_NONE = HIP_API_ID_NUMBER,\n') -for name in priv_lst: - f.write(' HIP_API_ID_' + name + ' = HIP_API_ID_NUMBER,\n') -f.write('};\n') - -# Generating the callbacks ID enumaration -f.write('\n// Return HIP API string\n') -f.write('static const char* hip_api_name(const uint32_t& id) {\n') -f.write(' switch(id) {\n') -for name in api_map.keys(): - f.write(' case HIP_API_ID_' + name + ': return "' + name + '";\n') -f.write(' };\n') -f.write(' return "unknown";\n') -f.write('};\n') - -# Generating the callbacks data structure -f.write('\n// HIP API callbacks data structure\n') -f.write( -'struct hip_api_data_t {\n' + -' uint64_t correlation_id;\n' + -' uint32_t phase;\n' + -' union {\n' -) -for name, args in api_map.items(): - if len(args) != 0: - f.write(' struct {\n') - for arg_tuple in args: - f.write(' ' + arg_tuple[0] + ' ' + arg_tuple[1] + ';\n') - f.write(' } ' + name + ';\n') -f.write( -' } args;\n' + -'};\n' -) - -# Generating the callbacks args data filling macros -f.write('\n// HIP API callbacks args data filling macros\n') -for name, args in api_map.items(): - f.write('#define INIT_' + name + '_CB_ARGS_DATA(cb_data) { \\\n') - if name in opts_map: - opts_list = opts_map[name] - if len(args) != len(opts_list): - print "Error: \"" + name + "\" API args and opts mismatch, args: ", args, ", opts: ", opts_list - for ind in range(0, len(args)): - arg_tuple = args[ind] -# arg_type = arg_tuple[0] - fld_name = arg_tuple[1] - arg_name = opts_list[ind] - f.write(' cb_data.args.' + name + '.' + fld_name + ' = ' + arg_name + '; \\\n') - f.write('};\n') -f.write('#define INIT_CB_ARGS_DATA(cb_id, cb_data) INIT_##cb_id##_CB_ARGS_DATA(cb_data)\n') - -# Generating the method for the API string, name and parameters -f.write('\n') -f.write('#if 0\n') -f.write('// HIP API string method, method name and parameters\n') -f.write('const char* hipApiString(hip_api_id_t id, const hip_api_data_t* data) {\n') -f.write(' std::ostringstream oss;\n') -f.write(' switch (id) {\n') -for name, args in api_map.items(): - f.write(' case HIP_API_ID_' + name + ':\n') - f.write(' oss << "' + name + '("') - for ind in range(0, len(args)): - arg_tuple = args[ind] - arg_name = arg_tuple[1] - if ind != 0: f.write(' << ","') - f.write('\n << " ' + arg_name + '=" << data->args.' + name + '.' + arg_name) - f.write('\n << ")";\n') - f.write(' break;\n') -f.write(' default: oss << "unknown";\n') -f.write(' };\n') -f.write(' return strdup(oss.str().c_str());\n') -f.write('};\n') -f.write('#endif\n') - -f.write('#endif // _HIP_PROF_STR_H\n'); - -print "Header '" + HEADER + "' is generated" -############################################################# +# Successfull exit +sys.exit(0)