From 663e42663b747513777f7de63e6b5e31cb96a05f Mon Sep 17 00:00:00 2001 From: Alex Sierra Date: Tue, 4 Apr 2023 13:01:56 -0500 Subject: [PATCH] core dump: SegmentBuilder classes added SegmentBuilder classes are used to get core dump data from the GPUs. So far, it uses thunk API calls and smaps to collect all data from the Hardware. Signed-off-by: Alex Sierra Change-Id: I2ad70ca5a951885181d3142653b186b0f6be739e [ROCm/ROCR-Runtime commit: 1083d5c35f8c51a976bff2ad9197cb72496f25db] --- .../runtime/hsa-runtime/CMakeLists.txt | 3 +- .../runtime/hsa-runtime/inc/amd_hsa_elf.h | 2 + .../libamdhsacode/amd_core_dump.cpp | 258 ++++++++++++++++++ 3 files changed, 262 insertions(+), 1 deletion(-) create mode 100644 projects/rocr-runtime/runtime/hsa-runtime/libamdhsacode/amd_core_dump.cpp diff --git a/projects/rocr-runtime/runtime/hsa-runtime/CMakeLists.txt b/projects/rocr-runtime/runtime/hsa-runtime/CMakeLists.txt index f92550f978..7bf93832c0 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/CMakeLists.txt +++ b/projects/rocr-runtime/runtime/hsa-runtime/CMakeLists.txt @@ -195,7 +195,8 @@ set ( SRCS core/util/lnx/os_linux.cpp libamdhsacode/amd_hsa_code_util.cpp libamdhsacode/amd_hsa_locks.cpp libamdhsacode/amd_options.cpp - libamdhsacode/amd_hsa_code.cpp ) + libamdhsacode/amd_hsa_code.cpp + libamdhsacode/amd_core_dump.cpp ) target_sources( ${CORE_RUNTIME_TARGET} PRIVATE ${SRCS} ) diff --git a/projects/rocr-runtime/runtime/hsa-runtime/inc/amd_hsa_elf.h b/projects/rocr-runtime/runtime/hsa-runtime/inc/amd_hsa_elf.h index d5359f18db..51aa389a06 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/inc/amd_hsa_elf.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/inc/amd_hsa_elf.h @@ -262,6 +262,8 @@ typedef enum { #define NT_AMD_HSA_PRODUCER_OPTIONS 5 #define NT_AMD_HSA_EXTENSION 6 #define NT_AMD_HSA_ISA_NAME 11 +/* AMDGPU snapshots of runtime, agent and queues state for use in core dump */ +#define NT_AMDGPU_CORE_STATE 33 #define NT_AMD_HSA_HLDEBUG_DEBUG 101 #define NT_AMD_HSA_HLDEBUG_TARGET 102 diff --git a/projects/rocr-runtime/runtime/hsa-runtime/libamdhsacode/amd_core_dump.cpp b/projects/rocr-runtime/runtime/hsa-runtime/libamdhsacode/amd_core_dump.cpp new file mode 100644 index 0000000000..91338a78ef --- /dev/null +++ b/projects/rocr-runtime/runtime/hsa-runtime/libamdhsacode/amd_core_dump.cpp @@ -0,0 +1,258 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2023, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// +#include +#include +#include +#include +#include +#include +#include +#include +#include "core/util/utils.h" +#include "./amd_hsa_code_util.hpp" +#include "hsakmt/hsakmt.h" + +constexpr char SNAPSHOT_INFO_ALIGNMENT = 0x8; + +namespace rocr { +namespace amd { +namespace coredump { +/* Implementation details */ +namespace impl { +class PackageBuilder { + public: + PackageBuilder() : st_(std::stringstream::out | std::stringstream::binary) {} + size_t Size() const { return st_.str().size(); } + template ::value>::type> + void Write(const T& v) { + st_.write((char*)&v, sizeof(T)); + } + void Write(const std::vector& v) { st_.write((const char*)v.data(), v.size()); } + void Write(void* data, uint32_t size) { st_.write((const char*)data, size); } + bool GetBuffer(void* out) { + size_t sz = Size(); + + if (!sz) return false; + std::memcpy(out, st_.str().c_str(), sz); + return true; + } + void Print(void* buf, uint64_t size) { + int i; + for (i = 0; i < size; i++) debug_print("%02x ", 0xFF & ((uint8_t*)buf)[i]); + debug_print("\n"); + } + private: + std::stringstream st_; +}; + +enum SegmentType { LOAD, NOTE }; +struct SegmentBuilder; + +struct SegmentInfo { + SegmentType stype; + uint64_t vaddr = 0; + uint64_t size = 0; + uint32_t flags = 0; + SegmentBuilder* builder; +}; + +using SegmentsInfo = std::vector; +using rocr::amd::hsa::alignUp; +struct SegmentBuilder { + virtual ~SegmentBuilder() = default; + /* Find which segments needs to be created. */ + virtual hsa_status_t Collect(SegmentsInfo& segments) = 0; + /* Called to read a given SegmentInfo's data. */ + virtual hsa_status_t Read(void* buf, size_t buf_size, off_t offset) = 0; +}; + +struct NoteSegmentBuilder : public SegmentBuilder { + hsa_status_t Collect(SegmentsInfo& segments) override { + void *runtime_ptr, *agents_ptr = NULL, *queues_ptr = NULL; + uint32_t runtime_size, agents_size, queue_size, n_entries, entry_size; + HsaVersionInfo versionInfo = {0}; + + if (hsaKmtDbgEnable(&runtime_ptr, &runtime_size)) return HSA_STATUS_ERROR; + std::unique_ptr runtime_info(runtime_ptr, std::free); + + if (hsaKmtGetVersion(&versionInfo)) return HSA_STATUS_ERROR; + /* Note version */ + note_package_builder_.Write(1); + /* Store version_major in PT_NOTE package */ + note_package_builder_.Write(versionInfo.KernelInterfaceMajorVersion); + /* Store version_minor in PT_NOTE package */ + note_package_builder_.Write(versionInfo.KernelInterfaceMinorVersion); + /* Store runtime_info_size in PT_NOTE package */ + note_package_builder_.Write(runtime_size); + + if (hsaKmtDbgGetDeviceData(&agents_ptr, &n_entries, &entry_size)) + return HSA_STATUS_ERROR; + agents_size = n_entries * entry_size; + std::unique_ptr agents_info(agents_ptr, std::free); + /* Store n_agents in PT_NOTE package */ + note_package_builder_.Write(n_entries); + /* Store agent_info_entry_size in PT_NOTE package */ + note_package_builder_.Write(entry_size); + + if (hsaKmtDbgGetQueueData(&queues_ptr, &n_entries, &entry_size, true)) + return HSA_STATUS_ERROR; + queue_size = n_entries * entry_size; + std::unique_ptr queues_info(queues_ptr, std::free); + /* Store n_queues in PT_NOTE package */ + note_package_builder_.Write(n_entries); + /* Store queue_info_entry_size in PT_NOTE package */ + note_package_builder_.Write(entry_size); + + PushInfo(runtime_info.get(), runtime_size); + PushInfo(agents_info.get(), agents_size); + PushInfo(queues_info.get(), queue_size); + if (hsaKmtDbgDisable()) return HSA_STATUS_ERROR; + + /* With note content, package this in the PT_NOTE. */ + PackageBuilder noteHeaderBuilder; + noteHeaderBuilder.Write (7); /* namesz */ + noteHeaderBuilder.Write (note_package_builder_.Size()); + noteHeaderBuilder.Write (NT_AMDGPU_CORE_STATE); /* type. */ + noteHeaderBuilder.Write ("AMDGPU\0"); + + raw_.resize(noteHeaderBuilder.Size() + note_package_builder_.Size()); + if (!noteHeaderBuilder.GetBuffer(raw_.data())) + return HSA_STATUS_ERROR; + if (!note_package_builder_.GetBuffer(&raw_[noteHeaderBuilder.Size()])) + return HSA_STATUS_ERROR; + + SegmentInfo s; + s.stype = NOTE; + s.vaddr = 0; + s.size = raw_.size(); + s.flags = 0; + s.builder = this; + segments.push_back(s); + + return HSA_STATUS_SUCCESS; + } + + hsa_status_t Read(void* buf, size_t buf_size, off_t offset) override { + if (offset + buf_size >raw_.size ()) return HSA_STATUS_ERROR; + memcpy(buf, raw_.data() + offset, buf_size); + return HSA_STATUS_SUCCESS; + } + + private: + PackageBuilder note_package_builder_; + std::vector raw_; + + void PushInfo(void *data, uint32_t size) { + note_package_builder_.Write(data, size); + size = alignUp(size, SNAPSHOT_INFO_ALIGNMENT) - size; + for (int i = 0; i < size; i++) + note_package_builder_.Write(0); + } +}; + +struct LoadSegmentBuilder : public SegmentBuilder { + LoadSegmentBuilder() : fd_(open("/proc/self/mem", O_RDONLY)) {} + + ~LoadSegmentBuilder() { + if (fd_ != -1) close(fd_); + } + + hsa_status_t Collect(SegmentsInfo& segments) override { + const std::string maps_path = "/proc/self/maps"; + std::ifstream maps(maps_path); + if (!maps.is_open()) { + fprintf(stderr, "Could not open '%s'", maps_path.c_str()); + return HSA_STATUS_ERROR; + } + + std::string line; + while (std::getline(maps, line)) { + std::istringstream isl{ line }; + std::string address, perms, offset, dev, inode, path; + if (!(isl >> address >> perms >> offset >> dev >> inode)) { + fprintf(stderr, "Failed to parse '%s'", maps_path.c_str()); + return HSA_STATUS_ERROR; + } + + std::getline(isl >> std::ws, path); + + /* Look for the /dev/dri/renderD* files. */ + if (path.rfind("/dev/dri/renderD", 0) == 0) { + uint64_t start, end; + if (sscanf(address.c_str(), "%lx-%lx", &start, &end) != 2) { + fprintf(stderr, "Failed to parse '%s'", maps_path.c_str()); + return HSA_STATUS_ERROR; + } + uint32_t flags = SHF_ALLOC; + flags |= (perms.find('w', 0) != std::string::npos) ? SHF_WRITE : 0; + flags |= (perms.find('x', 0) != std::string::npos) ? SHF_EXECINSTR : 0; + uint64_t size = end - start; + + debug_print("LOAD 0x%lx size: %ld\n", start, size); + SegmentInfo s; + s.stype = LOAD; + s.vaddr = start; + s.size = size; + s.flags = flags; + s.builder = this; + segments.push_back(s); + } + } + return HSA_STATUS_SUCCESS; + } + + hsa_status_t Read(void* buf, size_t buf_size, off_t offset) override { + if (fd_ == -1) return HSA_STATUS_ERROR; + if (pread(fd_, buf, buf_size, offset) == -1) { + perror("Failed to read GPU memory"); + return HSA_STATUS_ERROR; + } + return HSA_STATUS_SUCCESS; + } + + private: + int fd_ = -1; +}; +} // namespace impl +} // namespace coredump +} // namespace amd +} // namespace rocr