diff --git a/samples/api_buffered_tracing/client.cpp b/samples/api_buffered_tracing/client.cpp index 9dd5b13b00..b99e73f976 100644 --- a/samples/api_buffered_tracing/client.cpp +++ b/samples/api_buffered_tracing/client.cpp @@ -331,7 +331,8 @@ tool_tracing_callback(rocprofiler_context_id_t context, << ", " << as_hex(arg.page_migrate_end.end_addr) << ", " << arg.page_migrate_end.from_agent.handle << ", " << arg.page_migrate_end.to_agent.handle << ", " - << arg.page_migrate_end.trigger << ")"; + << arg.page_migrate_end.trigger << ", " << arg.page_migrate_end.error_code + << ")"; break; } case ROCPROFILER_PAGE_MIGRATION_PAGE_FAULT_START: @@ -373,6 +374,12 @@ tool_tracing_callback(rocprofiler_context_id_t context, << ")"; break; } + case ROCPROFILER_PAGE_MIGRATION_DROPPED_EVENT: + { + const auto& arg = record->args; + info << ", dropped_event=(" << arg.dropped_event.dropped_events_count << ")"; + break; + } case ROCPROFILER_PAGE_MIGRATION_NONE: case ROCPROFILER_PAGE_MIGRATION_LAST: { diff --git a/source/include/rocprofiler-sdk/cxx/serialization.hpp b/source/include/rocprofiler-sdk/cxx/serialization.hpp index 5c1e053b8c..ad318d546b 100644 --- a/source/include/rocprofiler-sdk/cxx/serialization.hpp +++ b/source/include/rocprofiler-sdk/cxx/serialization.hpp @@ -529,6 +529,7 @@ save(ArchiveT& ar, const rocprofiler_page_migration_page_migrate_end_t& data) ROCP_SDK_SAVE_DATA_FIELD(from_agent); ROCP_SDK_SAVE_DATA_FIELD(to_agent); ROCP_SDK_SAVE_DATA_FIELD(trigger); + ROCP_SDK_SAVE_DATA_FIELD(error_code); } template @@ -557,6 +558,13 @@ save(ArchiveT& ar, const rocprofiler_page_migration_unmap_from_gpu_t& data) ROCP_SDK_SAVE_DATA_FIELD(trigger); } +template +void +save(ArchiveT& ar, const rocprofiler_page_migration_dropped_event_t& data) +{ + ROCP_SDK_SAVE_DATA_FIELD(dropped_events_count); +} + namespace details { template @@ -582,6 +590,7 @@ ROCP_SDK_SPECIALIZE_PAGE_MIGRATION_ARG(PAGE_FAULT_END, page_fault_end) ROCP_SDK_SPECIALIZE_PAGE_MIGRATION_ARG(QUEUE_EVICTION, queue_eviction) ROCP_SDK_SPECIALIZE_PAGE_MIGRATION_ARG(QUEUE_RESTORE, queue_restore) ROCP_SDK_SPECIALIZE_PAGE_MIGRATION_ARG(UNMAP_FROM_GPU, unmap_from_gpu) +ROCP_SDK_SPECIALIZE_PAGE_MIGRATION_ARG(DROPPED_EVENT, dropped_event) #undef ROCP_SDK_SPECIALIZE_PAGE_MIGRATION_ARG diff --git a/source/include/rocprofiler-sdk/fwd.h b/source/include/rocprofiler-sdk/fwd.h index d1d32ec44c..103dacd4b2 100644 --- a/source/include/rocprofiler-sdk/fwd.h +++ b/source/include/rocprofiler-sdk/fwd.h @@ -338,6 +338,7 @@ typedef enum // NOLINT(performance-enum-size) ROCPROFILER_PAGE_MIGRATION_QUEUE_EVICTION, ROCPROFILER_PAGE_MIGRATION_QUEUE_RESTORE, ROCPROFILER_PAGE_MIGRATION_UNMAP_FROM_GPU, + ROCPROFILER_PAGE_MIGRATION_DROPPED_EVENT, ROCPROFILER_PAGE_MIGRATION_LAST, } rocprofiler_page_migration_operation_t; diff --git a/source/include/rocprofiler-sdk/kfd/page_migration_args.h b/source/include/rocprofiler-sdk/kfd/page_migration_args.h index d1eed33d9f..9d322660f5 100644 --- a/source/include/rocprofiler-sdk/kfd/page_migration_args.h +++ b/source/include/rocprofiler-sdk/kfd/page_migration_args.h @@ -52,6 +52,7 @@ typedef struct rocprofiler_page_migration_page_migrate_end_t rocprofiler_agent_id_t from_agent; rocprofiler_agent_id_t to_agent; rocprofiler_page_migration_trigger_t trigger; + int32_t error_code; } rocprofiler_page_migration_page_migrate_end_t; typedef struct rocprofiler_page_migration_page_fault_start_t @@ -88,6 +89,11 @@ typedef struct rocprofiler_page_migration_unmap_from_gpu_t rocprofiler_page_migration_unmap_from_gpu_trigger_t trigger; } rocprofiler_page_migration_unmap_from_gpu_t; +typedef struct rocprofiler_page_migration_dropped_event_t +{ + uint32_t dropped_events_count; +} rocprofiler_page_migration_dropped_event_t; + typedef union { rocprofiler_page_migration_none_t none; @@ -98,6 +104,7 @@ typedef union rocprofiler_page_migration_queue_eviction_t queue_eviction; rocprofiler_page_migration_queue_restore_t queue_restore; rocprofiler_page_migration_unmap_from_gpu_t unmap_from_gpu; + rocprofiler_page_migration_dropped_event_t dropped_event; uint64_t reserved[16]; } rocprofiler_page_migration_args_t; diff --git a/source/lib/rocprofiler-sdk/details/kfd_ioctl.h b/source/lib/rocprofiler-sdk/details/kfd_ioctl.h index bd69ad06bb..32ca4b76a6 100644 --- a/source/lib/rocprofiler-sdk/details/kfd_ioctl.h +++ b/source/lib/rocprofiler-sdk/details/kfd_ioctl.h @@ -578,6 +578,7 @@ enum kfd_smi_event KFD_SMI_EVENT_QUEUE_EVICTION = 9, KFD_SMI_EVENT_QUEUE_RESTORE = 10, KFD_SMI_EVENT_UNMAP_FROM_GPU = 11, + KFD_SMI_EVENT_DROPPED_EVENT = 12, /* * max event number, as a flag bit to get events from all processes, @@ -590,27 +591,27 @@ enum kfd_smi_event enum KFD_MIGRATE_TRIGGERS { - KFD_MIGRATE_TRIGGER_PREFETCH, - KFD_MIGRATE_TRIGGER_PAGEFAULT_GPU, - KFD_MIGRATE_TRIGGER_PAGEFAULT_CPU, - KFD_MIGRATE_TRIGGER_TTM_EVICTION + KFD_MIGRATE_TRIGGER_PREFETCH, /* Prefetch to GPU */ + KFD_MIGRATE_TRIGGER_PAGEFAULT_GPU, /* GPU page fault recover */ + KFD_MIGRATE_TRIGGER_PAGEFAULT_CPU, /* CPU page fault recover */ + KFD_MIGRATE_TRIGGER_TTM_EVICTION /* TTM eviction */ }; enum KFD_QUEUE_EVICTION_TRIGGERS { - KFD_QUEUE_EVICTION_TRIGGER_SVM, - KFD_QUEUE_EVICTION_TRIGGER_USERPTR, - KFD_QUEUE_EVICTION_TRIGGER_TTM, - KFD_QUEUE_EVICTION_TRIGGER_SUSPEND, - KFD_QUEUE_EVICTION_CRIU_CHECKPOINT, - KFD_QUEUE_EVICTION_CRIU_RESTORE + KFD_QUEUE_EVICTION_TRIGGER_SVM, /* SVM buffer migration */ + KFD_QUEUE_EVICTION_TRIGGER_USERPTR, /* userptr movement */ + KFD_QUEUE_EVICTION_TRIGGER_TTM, /* TTM move buffer */ + KFD_QUEUE_EVICTION_TRIGGER_SUSPEND, /* GPU suspend */ + KFD_QUEUE_EVICTION_CRIU_CHECKPOINT, /* CRIU checkpoint */ + KFD_QUEUE_EVICTION_CRIU_RESTORE /* CRIU restore */ }; enum KFD_SVM_UNMAP_TRIGGERS { - KFD_SVM_UNMAP_TRIGGER_MMU_NOTIFY, - KFD_SVM_UNMAP_TRIGGER_MMU_NOTIFY_MIGRATE, - KFD_SVM_UNMAP_TRIGGER_UNMAP_FROM_CPU + KFD_SVM_UNMAP_TRIGGER_MMU_NOTIFY, /* MMU notifier CPU buffer movement */ + KFD_SVM_UNMAP_TRIGGER_MMU_NOTIFY_MIGRATE, /* MMU notifier page migration */ + KFD_SVM_UNMAP_TRIGGER_UNMAP_FROM_CPU /* Unmap to free the buffer */ }; #define KFD_SMI_EVENT_MASK_FROM_INDEX(i) (1ULL << ((i) -1)) @@ -699,6 +700,68 @@ struct kfd_ioctl_spm_args __u32 has_data_loss; }; +/* + * SVM event tracing via SMI system management interface + * + * Open event file descriptor + * use ioctl AMDKFD_IOC_SMI_EVENTS, pass in gpuid and return a anonymous file + * descriptor to receive SMI events. + * If calling with sudo permission, then file descriptor can be used to receive + * SVM events from all processes, otherwise, to only receive SVM events of same + * process. + * + * To enable the SVM event + * Write event file descriptor with KFD_SMI_EVENT_MASK_FROM_INDEX(event) bitmap + * mask to start record the event to the kfifo, use bitmap mask combination + * for multiple events. New event mask will overwrite the previous event mask. + * KFD_SMI_EVENT_MASK_FROM_INDEX(KFD_SMI_EVENT_ALL_PROCESS) bit requires sudo + * permisson to receive SVM events from all process. + * + * To receive the event + * Application can poll file descriptor to wait for the events, then read event + * from the file into a buffer. Each event is one line string message, starting + * with the event id, then the event specific information. + * + * To decode event information + * The following event format string macro can be used with sscanf to decode + * the specific event information. + * event triggers: the reason to generate the event, defined as enum for unmap, + * eviction and migrate events. + * node, from, to, prefetch_loc, preferred_loc: GPU ID, or 0 for system memory. + * addr: user mode address, in pages + * size: in pages + * pid: the process ID to generate the event + * ns: timestamp in nanosecond-resolution, starts at system boot time but + * stops during suspend + * migrate_update: GPU page fault is recovered by 'M' for migrate, 'U' for update + * rw: 'W' for write page fault, 'R' for read page fault + * rescheduled: 'R' if the queue restore failed and rescheduled to try again + * error_code: migrate failure error code, 0 if no error + * drop_count: how many events dropped when fifo is full + */ +#define KFD_EVENT_FMT_UPDATE_GPU_RESET(reset_seq_num, reset_cause) \ + "%x %s\n", (reset_seq_num), (reset_cause) +#define KFD_EVENT_FMT_THERMAL_THROTTLING(bitmask, counter) "%llx:%llx\n", (bitmask), (counter) +#define KFD_EVENT_FMT_VMFAULT(pid, task_name) "%x:%s\n", (pid), (task_name) +#define KFD_EVENT_FMT_PAGEFAULT_START(ns, pid, addr, node, rw) \ + "%lld -%d @%lx(%x) %c\n", (ns), (pid), (addr), (node), (rw) +#define KFD_EVENT_FMT_PAGEFAULT_END(ns, pid, addr, node, migrate_update) \ + "%lld -%d @%lx(%x) %c\n", (ns), (pid), (addr), (node), (migrate_update) +#define KFD_EVENT_FMT_MIGRATE_START( \ + ns, pid, start, size, from, to, prefetch_loc, preferred_loc, migrate_trigger) \ + "%lld -%d @%lx(%lx) %x->%x %x:%x %d\n", (ns), (pid), (start), (size), (from), (to), \ + (prefetch_loc), (preferred_loc), (migrate_trigger) +#define KFD_EVENT_FMT_MIGRATE_END(ns, pid, start, size, from, to, migrate_trigger, error_code) \ + "%lld -%d @%lx(%lx) %x->%x %d %d\n", (ns), (pid), (start), (size), (from), (to), \ + (migrate_trigger), (error_code) +#define KFD_EVENT_FMT_QUEUE_EVICTION(ns, pid, node, evict_trigger) \ + "%lld -%d %x %d\n", (ns), (pid), (node), (evict_trigger) +#define KFD_EVENT_FMT_QUEUE_RESTORE(ns, pid, node, rescheduled) \ + "%lld -%d %x %c\n", (ns), (pid), (node), (rescheduled) +#define KFD_EVENT_FMT_UNMAP_FROM_GPU(ns, pid, addr, size, node, unmap_trigger) \ + "%lld -%d @%lx(%lx) %x %d\n", (ns), (pid), (addr), (size), (node), (unmap_trigger) +#define KFD_EVENT_FMT_DROPPED_EVENT(ns, pid, drop_count) "%lld -%d %d\n", (ns), (pid), (drop_count) + /************************************************************************************************** * CRIU IOCTLs (Checkpoint Restore In Userspace) * diff --git a/source/lib/rocprofiler-sdk/page_migration/abi.cpp b/source/lib/rocprofiler-sdk/page_migration/abi.cpp index 6293474040..c9f2e415c2 100644 --- a/source/lib/rocprofiler-sdk/page_migration/abi.cpp +++ b/source/lib/rocprofiler-sdk/page_migration/abi.cpp @@ -50,11 +50,12 @@ static_assert(KFD_SMI_EVENT_PAGE_FAULT_END == 8); static_assert(KFD_SMI_EVENT_QUEUE_EVICTION == 9); static_assert(KFD_SMI_EVENT_QUEUE_RESTORE == 10); static_assert(KFD_SMI_EVENT_UNMAP_FROM_GPU == 11); +static_assert(KFD_SMI_EVENT_DROPPED_EVENT == 12); static_assert(KFD_SMI_EVENT_ALL_PROCESS == 64); // Update page_migration.def.cpp with event mappings // Update page_migration.cpp to parse and report new event -static_assert(ROCPROFILER_PAGE_MIGRATION_LAST == 8, +static_assert(ROCPROFILER_PAGE_MIGRATION_LAST == 9, "New event added, update KFD to ROCPROFILER mappings"); // clang-format off diff --git a/source/lib/rocprofiler-sdk/page_migration/page_migration.cpp b/source/lib/rocprofiler-sdk/page_migration/page_migration.cpp index 50cb702b25..0afb670d13 100644 --- a/source/lib/rocprofiler-sdk/page_migration/page_migration.cpp +++ b/source/lib/rocprofiler-sdk/page_migration/page_migration.cpp @@ -260,8 +260,9 @@ parse_event(std::string_view str) auto& e = rec.args.page_migrate_end; uint32_t kind{}; uint32_t trigger{}; - uint32_t _from_node = 0; - uint32_t _to_node = 0; + uint32_t _from_node = 0; + uint32_t _to_node = 0; + int32_t _error_code = 0; std::sscanf(str.data(), page_migration_info::format_str.data(), @@ -272,7 +273,8 @@ parse_event(std::string_view str) &e.end_addr, &_from_node, &_to_node, - &trigger); + &trigger, + &_error_code); e.end_addr += e.start_addr; e.trigger = static_cast(trigger); @@ -281,15 +283,20 @@ parse_event(std::string_view str) e.from_agent = get_node_agent_id(_from_node); e.to_agent = get_node_agent_id(_to_node); + // For older kernel versions, no event is generated in the error case, + // so the default value of 0 is correct for any given end event. + e.error_code = _error_code; + ROCP_TRACE << fmt::format("Page migrate end [ ts: {} pid: {} addr s: 0x{:X} addr e: " - "0x{:X} from node: {} to node: {} trigger: {} ] \n", + "0x{:X} from node: {} to node: {} trigger: {} error code: {}] \n", rec.timestamp, rec.pid, e.start_addr, e.end_addr, e.from_agent.handle, e.to_agent.handle, - trigger); + trigger, + _error_code); return rec; } @@ -391,6 +398,29 @@ parse_event(std::string_view str) return rec; } +template <> +page_migration_record_t +parse_event(std::string_view str) +{ + auto rec = page_migration_record_t{}; + auto& e = rec.args.dropped_event; + uint32_t kind{}; + + std::sscanf(str.data(), + page_migration_info::format_str.data(), + &kind, + &rec.timestamp, + &rec.pid, + &e.dropped_events_count); + + ROCP_TRACE << fmt::format("Dropped events [ ts: {} pid: {} dropped count: {} ] \n", + rec.timestamp, + rec.pid, + e.dropped_events_count); + + return rec; +} + template <> page_migration_record_t parse_event(std::string_view) { @@ -853,13 +883,16 @@ rocprofiler_status_t init() { // Testing page migration - return init({ROCPROFILER_PAGE_MIGRATION_PAGE_MIGRATE_START, - ROCPROFILER_PAGE_MIGRATION_PAGE_MIGRATE_END, - ROCPROFILER_PAGE_MIGRATION_PAGE_FAULT_START, - ROCPROFILER_PAGE_MIGRATION_PAGE_FAULT_END, - ROCPROFILER_PAGE_MIGRATION_QUEUE_EVICTION, - ROCPROFILER_PAGE_MIGRATION_QUEUE_RESTORE, - ROCPROFILER_PAGE_MIGRATION_UNMAP_FROM_GPU}); + return init({ + ROCPROFILER_PAGE_MIGRATION_PAGE_MIGRATE_START, + ROCPROFILER_PAGE_MIGRATION_PAGE_MIGRATE_END, + ROCPROFILER_PAGE_MIGRATION_PAGE_FAULT_START, + ROCPROFILER_PAGE_MIGRATION_PAGE_FAULT_END, + ROCPROFILER_PAGE_MIGRATION_QUEUE_EVICTION, + ROCPROFILER_PAGE_MIGRATION_QUEUE_RESTORE, + ROCPROFILER_PAGE_MIGRATION_UNMAP_FROM_GPU, + ROCPROFILER_PAGE_MIGRATION_DROPPED_EVENT, + }); } void diff --git a/source/lib/rocprofiler-sdk/page_migration/page_migration.def.cpp b/source/lib/rocprofiler-sdk/page_migration/page_migration.def.cpp index ab726c4bae..abffb38d17 100644 --- a/source/lib/rocprofiler-sdk/page_migration/page_migration.def.cpp +++ b/source/lib/rocprofiler-sdk/page_migration/page_migration.def.cpp @@ -45,12 +45,13 @@ using trigger_type_list_t = common::mpl::type_list%x %x:%x %d\n"); -SPECIALIZE_PAGE_MIGRATION_INFO(PAGE_MIGRATE_END, MIGRATE_END, "%x %ld -%d @%lx(%lx) %x->%x %d\n" ); +SPECIALIZE_PAGE_MIGRATION_INFO(PAGE_MIGRATE_END, MIGRATE_END, "%x %ld -%d @%lx(%lx) %x->%x %d %d\n" ); SPECIALIZE_PAGE_MIGRATION_INFO(PAGE_FAULT_START, PAGE_FAULT_START, "%x %ld -%d @%lx(%x) %c\n" ); SPECIALIZE_PAGE_MIGRATION_INFO(PAGE_FAULT_END, PAGE_FAULT_END, "%x %ld -%d @%lx(%x) %c\n" ); SPECIALIZE_PAGE_MIGRATION_INFO(QUEUE_EVICTION, QUEUE_EVICTION, "%x %ld -%d %x %d\n" ); SPECIALIZE_PAGE_MIGRATION_INFO(QUEUE_RESTORE, QUEUE_RESTORE, "%x %ld -%d %x\n" ); SPECIALIZE_PAGE_MIGRATION_INFO(UNMAP_FROM_GPU, UNMAP_FROM_GPU, "%x %ld -%d @%lx(%lx) %x %d\n" ); +SPECIALIZE_PAGE_MIGRATION_INFO(DROPPED_EVENT, DROPPED_EVENT, "%x %ld -%d %d\n" ); #undef SPECIALIZE_PAGE_MIGRATION_INFO // clang-format on diff --git a/source/lib/rocprofiler-sdk/tests/page_migration.cpp b/source/lib/rocprofiler-sdk/tests/page_migration.cpp index e2f0ca31cb..da38b3619a 100644 --- a/source/lib/rocprofiler-sdk/tests/page_migration.cpp +++ b/source/lib/rocprofiler-sdk/tests/page_migration.cpp @@ -71,7 +71,7 @@ TEST(page_migration, readlines) parse_lines(); } -TEST(page_migtation, rocprof_kfd_map) +TEST(page_migration, rocprof_kfd_map) { using namespace rocprofiler::page_migration; using namespace rocprofiler::common::container; @@ -101,5 +101,6 @@ TEST(page_migtation, rocprof_kfd_map) EXPECT_EQ(to_kfd_str(KFD_SMI_EVENT_QUEUE_EVICTION), ROCPROFILER_PAGE_MIGRATION_QUEUE_EVICTION); EXPECT_EQ(to_kfd_str(KFD_SMI_EVENT_QUEUE_RESTORE), ROCPROFILER_PAGE_MIGRATION_QUEUE_RESTORE); EXPECT_EQ(to_kfd_str(KFD_SMI_EVENT_UNMAP_FROM_GPU), ROCPROFILER_PAGE_MIGRATION_UNMAP_FROM_GPU); + EXPECT_EQ(to_kfd_str(KFD_SMI_EVENT_DROPPED_EVENT), ROCPROFILER_PAGE_MIGRATION_DROPPED_EVENT); // clang-format on } diff --git a/tests/page-migration/validate.py b/tests/page-migration/validate.py index 15619d96b2..4b8ca93742 100644 --- a/tests/page-migration/validate.py +++ b/tests/page-migration/validate.py @@ -300,7 +300,7 @@ def test_page_migration_data(input_data): for op_name in bf_op_names: assert "PAGE_MIGRATION" in op_name - assert len(bf_op_names) == 8 + assert len(bf_op_names) == 9 nodes = set(x.id.handle for x in sdk_data.agents) allocations = get_allocated_pages(callback_records) @@ -379,6 +379,11 @@ def test_page_migration_data(input_data): validate_node(arg.agent_id, nodes) assert 0 <= arg.trigger < 3 + if "dropped_event" in r: + arg = r.dropped_event + # We shouldn't get any dropped events. If we do, our test needs to be redesigned. + assert arg.dropped_events_count == 0 + if __name__ == "__main__": exit_code = pytest.main(["-x", __file__] + sys.argv[1:])