From d209a2fdd158aed0cc54a722c905546c07be0306 Mon Sep 17 00:00:00 2001 From: Giovanni LB Date: Tue, 11 Jul 2023 20:10:29 -0300 Subject: [PATCH] Added error handling to att iterate_data. Fix for genasm. Change-Id: Ia86e629e74c6e00b98155355beabf69681a88875 [ROCm/rocprofiler commit: 1c11700521498a0bb46b3651e649892153cf969b] --- projects/rocprofiler/plugin/att/drawing.py | 2 +- projects/rocprofiler/plugin/att/stitch.py | 6 ++-- projects/rocprofiler/plugin/att/trace_view.py | 6 ++-- .../rocprofiler/src/core/hsa/queues/queue.cpp | 31 +++++++++++-------- 4 files changed, 25 insertions(+), 20 deletions(-) diff --git a/projects/rocprofiler/plugin/att/drawing.py b/projects/rocprofiler/plugin/att/drawing.py index cd44fc3219..a5702410fc 100644 --- a/projects/rocprofiler/plugin/att/drawing.py +++ b/projects/rocprofiler/plugin/att/drawing.py @@ -133,7 +133,7 @@ def draw_wave_states(selections, normalize, TIMELINES): kernel = np.asarray([np.exp(-abs(10*k/kernsize)) for k in range(-kernsize//2,kernsize//2+1)]) kernel /= np.sum(kernel) - timelines = [np.convolve(time, kernel)[kernsize//2:-kernsize//2] for time in timelines] + timelines = [np.convolve(time, kernel)[kernsize//2:-kernsize//2] for time in timelines if len(time) > 0] [plt.plot(cycles, t, label='State '+s, linewidth=1.1, color=c) for t, s, c, sel in zip(timelines, STATES, colors, selections) if sel] diff --git a/projects/rocprofiler/plugin/att/stitch.py b/projects/rocprofiler/plugin/att/stitch.py index 03f2875913..46907306d7 100644 --- a/projects/rocprofiler/plugin/att/stitch.py +++ b/projects/rocprofiler/plugin/att/stitch.py @@ -332,7 +332,7 @@ def stitch(insts, raw_code, jumps, gfxv): elif inst[1] == IMMED and 's_waitcnt ' in as_line[0]: if 'lgkmcnt' in as_line[0]: wait_N = int(as_line[0].split('lgkmcnt(')[1].split(')')[0]) - flight_count.append([as_line[-1], num_inflight, wait_N]) + flight_count.append([as_line[5], num_inflight, wait_N]) if wait_N == 0: smem_ordering = 0 if smem_ordering == 0: @@ -349,7 +349,7 @@ def stitch(insts, raw_code, jumps, gfxv): if 'vmcnt' in as_line[0]: wait_N = int(as_line[0].split('vmcnt(')[1].split(')')[0]) - flight_count.append([as_line[-1], num_inflight, wait_N]) + flight_count.append([as_line[5], num_inflight, wait_N]) if wait_N == 0: vlmem_ordering = 0 if vlmem_ordering == 0: @@ -369,7 +369,7 @@ def stitch(insts, raw_code, jumps, gfxv): wait_N = int(as_line[0].split('vscnt(')[1].split(')')[0]) except: wait_N = int(as_line[0].split('vmcnt(')[1].split(')')[0]) - flight_count.append([as_line[-1], num_inflight, wait_N]) + flight_count.append([as_line[5], num_inflight, wait_N]) if wait_N == 0: vsmem_ordering = 0 if vsmem_ordering == 0: diff --git a/projects/rocprofiler/plugin/att/trace_view.py b/projects/rocprofiler/plugin/att/trace_view.py index ae0f5ce474..ac61713b73 100755 --- a/projects/rocprofiler/plugin/att/trace_view.py +++ b/projects/rocprofiler/plugin/att/trace_view.py @@ -280,9 +280,6 @@ def view_trace(args, code, dbnames, att_filenames, bReturnLoc, OCCUPANCY, bDumpO if mpi_root: JSON_GLOBAL_DICTIONARY['code.json'] = Readable({"code": code[:allse_maxline+16], "top_n": get_top_n(code[:allse_maxline+16])}) - if bReturnLoc: - return flight_count - for key in simd_wave_filenames.keys(): wv_array = [[ int(s[0].split('_sm')[1].split('_sl')[0]), @@ -324,6 +321,9 @@ def view_trace(args, code, dbnames, att_filenames, bReturnLoc, OCCUPANCY, bDumpO for k, v in return_dict.items(): JSON_GLOBAL_DICTIONARY[k] = v + if bReturnLoc: + return flight_count + if bDumpOnly == False: if MPI_COMM is not None: JSON_GLOBAL_DICTIONARY = MPI_COMM.gather(JSON_GLOBAL_DICTIONARY, root=0) diff --git a/projects/rocprofiler/src/core/hsa/queues/queue.cpp b/projects/rocprofiler/src/core/hsa/queues/queue.cpp index 780e5aaa40..10d4f3da0e 100644 --- a/projects/rocprofiler/src/core/hsa/queues/queue.cpp +++ b/projects/rocprofiler/src/core/hsa/queues/queue.cpp @@ -348,7 +348,14 @@ void AddAttRecord(rocprofiler_record_att_tracer_t* record, hsa_agent_t gpu_agent att_pending_signal_t& pending) { Agent::AgentInfo agent_info = hsa_support::GetAgentInfo(gpu_agent.handle); att_trace_callback_data_t data; - hsa_ven_amd_aqlprofile_iterate_data(pending.profile, attTraceDataCallback, &data); + hsa_status_t status = hsa_ven_amd_aqlprofile_iterate_data(pending.profile, attTraceDataCallback, &data); + + if ((status & HSA_STATUS_ERROR_OUT_OF_RESOURCES) == HSA_STATUS_ERROR_OUT_OF_RESOURCES) + rocprofiler::warning("Warning: ATT buffer full!\n"); + if ((status & HSA_STATUS_ERROR_EXCEPTION) == HSA_STATUS_ERROR_EXCEPTION) + rocprofiler::warning("Warning: ATT received a UTC memory error!\n"); + if (status == HSA_STATUS_ERROR) + fatal("Thread Trace Error!"); // Allocate memory for shader_engine_data record->shader_engine_data = static_cast( @@ -702,7 +709,7 @@ std::pair, bool> GetAllowedProfilesList(const void* packets, i for (const std::string& kernel_matches : kernel_profile_names) if (kernel_name.find(kernel_matches) != std::string::npos) b_profile_this_object = true; } catch (...) { - printf("Warning: Unknown name for object %lu\n", kdispatch.kernel_object); + rocprofiler::warning("Warning: Unknown name for object %lu\n", kdispatch.kernel_object); } current_writer_id += 1; } @@ -745,20 +752,18 @@ hsa_ven_amd_aqlprofile_profile_t* ProcessATTParams(Packet::packet_t& start_packe for (const std::string& counter_name : att_counters_names) { const Metric* metric = metrics_dict_->Get(counter_name); const BaseMetric* base = dynamic_cast(metric); - if (!base) { - printf("Invalid base metric value: %s\n", counter_name.c_str()); - exit(1); - } + if (!base) rocprofiler::fatal("Invalid base metric value: %s\n", counter_name.c_str()); + std::vector counters; base->GetCounters(counters); hsa_ven_amd_aqlprofile_event_t event = counters[0]->event; - if (event.block_name != HSA_VEN_AMD_AQLPROFILE_BLOCK_NAME_SQ) { - printf("Only events from the SQ block can be selected for ATT."); - exit(1); - } - att_params.push_back( - {static_cast(int(ROCPROFILER_ATT_PERFCOUNTER)), - event.counter_id | (event.counter_id ? (0xF << 24) : 0)}); + if (event.block_name != HSA_VEN_AMD_AQLPROFILE_BLOCK_NAME_SQ) + rocprofiler::fatal("Only events from the SQ block can be selected for ATT.\n"); + + att_params.push_back({ + static_cast(int(ROCPROFILER_ATT_PERFCOUNTER)), + event.counter_id | (event.counter_id ? (0xF << 24) : 0) + }); num_att_counters += 1; }