From 938b34da24d398ce2ffae2b89b3a3f66d2271ffe Mon Sep 17 00:00:00 2001 From: jordans Date: Tue, 11 Mar 2025 16:56:02 -0400 Subject: [PATCH] hsakmt: Initial Commit for the HSA KMT Model The over arching goal it so provide an API that pre-silicon models can latch into for software bring up.# Please enter the commit message for your changes. Lines starting [ROCm/ROCR-Runtime commit: d4b85b6bf5babc8de7c70c137e528600cf8f7143] --- .../rocr-runtime/libhsakmt/CMakeLists.txt | 9 + .../libhsakmt/include/hsakmt/hsakmt.h | 17 + .../libhsakmt/include/hsakmt/hsakmtmodel.h | 36 + .../include/hsakmt/hsakmtmodeliface.h | 109 +++ projects/rocr-runtime/libhsakmt/src/debug.c | 2 +- projects/rocr-runtime/libhsakmt/src/events.c | 6 +- projects/rocr-runtime/libhsakmt/src/fmm.c | 18 + .../rocr-runtime/libhsakmt/src/hsakmtmodel.c | 823 ++++++++++++++++++ .../rocr-runtime/libhsakmt/src/libhsakmt.c | 4 + .../rocr-runtime/libhsakmt/src/libhsakmt.h | 2 + projects/rocr-runtime/libhsakmt/src/memory.c | 2 - .../rocr-runtime/libhsakmt/src/openclose.c | 11 +- .../rocr-runtime/libhsakmt/src/topology.c | 47 +- .../core/driver/kfd/amd_kfd_driver.cpp | 9 + .../core/driver/xdna/amd_xdna_driver.cpp | 6 + .../hsa-runtime/core/inc/amd_kfd_driver.h | 2 + .../hsa-runtime/core/inc/amd_xdna_driver.h | 2 + .../runtime/hsa-runtime/core/inc/driver.h | 4 + .../core/runtime/amd_gpu_agent.cpp | 20 +- .../hsa-runtime/core/runtime/amd_topology.cpp | 3 + 20 files changed, 1104 insertions(+), 28 deletions(-) create mode 100644 projects/rocr-runtime/libhsakmt/include/hsakmt/hsakmtmodel.h create mode 100644 projects/rocr-runtime/libhsakmt/include/hsakmt/hsakmtmodeliface.h create mode 100644 projects/rocr-runtime/libhsakmt/src/hsakmtmodel.c diff --git a/projects/rocr-runtime/libhsakmt/CMakeLists.txt b/projects/rocr-runtime/libhsakmt/CMakeLists.txt index fa49dc5441..cd4e16c7d5 100644 --- a/projects/rocr-runtime/libhsakmt/CMakeLists.txt +++ b/projects/rocr-runtime/libhsakmt/CMakeLists.txt @@ -116,6 +116,7 @@ set ( HSAKMT_SRC "src/debug.c" "src/events.c" "src/fmm.c" "src/globals.c" + "src/hsakmtmodel.c" "src/libhsakmt.c" "src/memory.c" "src/openclose.c" @@ -186,6 +187,14 @@ target_link_libraries ( ${HSAKMT_TARGET} target_compile_options(${HSAKMT_TARGET} PRIVATE ${DRM_CFLAGS} ${HSAKMT_C_FLAGS}) +include(CheckFunctionExists) +set(CMAKE_REQUIRED_DEFINITIONS -D__USE_GNU=1) +set(CMAKE_REQUIRED_INCLUDES sys/mman.h) +check_function_exists(memfd_create HAVE_MEMFD_CREATE) +if(HAVE_MEMFD_CREATE) + target_compile_definitions(${HSAKMT_TARGET} PRIVATE -DHAVE_MEMFD_CREATE=1) +endif() + ## Define default paths and packages. if( CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT ) set ( CMAKE_INSTALL_PREFIX "/opt/rocm" ) diff --git a/projects/rocr-runtime/libhsakmt/include/hsakmt/hsakmt.h b/projects/rocr-runtime/libhsakmt/include/hsakmt/hsakmt.h index 8361a3cd94..787e29275d 100644 --- a/projects/rocr-runtime/libhsakmt/include/hsakmt/hsakmt.h +++ b/projects/rocr-runtime/libhsakmt/include/hsakmt/hsakmt.h @@ -1208,6 +1208,23 @@ hsaKmtPcSamplingStop( HsaPcSamplingTraceId traceId ); +/** + * Check if the HSA KMT Model is enabled + * + * Arguments: + * @enable (OUT) - true if the HSA KMT Model is enabled, false otherwise + * + * Return: + * HSAKMT_STATUS_ERROR - failed + * HSAKMT_STATUS_SUCCESS - successfully complete + */ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtModelEnabled( + bool* enable // OUT +); + #ifdef __cplusplus } //extern "C" #endif diff --git a/projects/rocr-runtime/libhsakmt/include/hsakmt/hsakmtmodel.h b/projects/rocr-runtime/libhsakmt/include/hsakmt/hsakmtmodel.h new file mode 100644 index 0000000000..e830453886 --- /dev/null +++ b/projects/rocr-runtime/libhsakmt/include/hsakmt/hsakmtmodel.h @@ -0,0 +1,36 @@ +/* + * Copyright © 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _HSAKMTMODEL_H_ +#define _HSAKMTMODEL_H_ +#include +extern bool hsakmt_use_model; +extern char *hsakmt_model_topology; +void model_init_env_vars(void); +void model_init(void); +void model_set_mmio_page(void *ptr); +void model_set_event_page(void *ptr, unsigned event_limit); +int model_kfd_ioctl(unsigned long request, void *arg); +#endif /* _HSAKMTMODEL_H_ */ \ No newline at end of file diff --git a/projects/rocr-runtime/libhsakmt/include/hsakmt/hsakmtmodeliface.h b/projects/rocr-runtime/libhsakmt/include/hsakmt/hsakmtmodeliface.h new file mode 100644 index 0000000000..50f151c3cf --- /dev/null +++ b/projects/rocr-runtime/libhsakmt/include/hsakmt/hsakmtmodeliface.h @@ -0,0 +1,109 @@ +/* + * Copyright © 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _HSAKMTMODELIFACE_H_ +#define _HSAKMTMODELIFACE_H_ + +#include + +// Changelog: +// 0.2: Add set_set_event function to hsakmt_model_functions +#define HSAKMT_MODEL_INTERFACE_VERSION_MAJOR 0 +#define HSAKMT_MODEL_INTERFACE_VERSION_MINOR 4 + +typedef struct hsakmt_model hsakmt_model_t; +typedef struct hsakmt_model_queue hsakmt_model_queue_t; + +// Description of a queue to be registered with the model. +// +// Addresses are relative to the global aperture. +struct hsakmt_model_queue_info { + uint64_t ring_base_address; + uint64_t write_pointer_address; + uint64_t read_pointer_address; + + uint64_t *doorbell; + + uint32_t ring_size; // in bytes + uint32_t queue_type; +}; + +// Pointer to a "set event" function. +// +// data is a user-provided opaque pointer. +// event_id is the ID of the event to set (as in amd_signal_s::event_id). +typedef void (*hsakmt_model_set_event_fn)(void *data, unsigned event_id); + +// Interface provided by the software model implementation. +// +// Queried from a shared library by calling an export called +// `get_hsakmt_model_functions` +// +// Interface versioning follows the semantic versioning model: clients that +// know about interface version X.Y can use any implementation that provides +// version X.Z with Z >= Y. +// +// The model is designed to support only one VMID space. +struct hsakmt_model_functions { + uint32_t version_major; // HSAKMT_MODEL_INTERFACE_VERSION_MAJOR + uint32_t version_minor; // HSAKMT_MODEL_INTERFACE_VERSION_MINOR + + // Create a GPU device model. + hsakmt_model_t *(*create)(void); + + // Destroy a GPU device model. + void (*destroy)(hsakmt_model_t *model); + + // Set the global aperture. GPU virtual address 0 is at CPU address `base`. + void (*set_global_aperture)(hsakmt_model_t *model, void *base, uint64_t size); + void (*alloced_memory)(hsakmt_model_t *model, void *base, uint64_t size, uint32_t flags); + void (*freed_memory)(hsakmt_model_t *model, void *base, uint64_t size); + // Register a callback that the model should call when an event is signaled. + // `data` is client data that is opaque to the model. + // + // TODO: Deprecated -- remove this! + void (*set_notify_event)(hsakmt_model_t *model, void (*callback)(void *data), void *data); + + // Register a callback that the model should call in order to wait for an + // event to be signaled. + // `data` is client data that is opaque to the model. + void (*set_wait_event)(hsakmt_model_t *model, void (*callback)(void *data, uint64_t address, uint64_t age), void *data); + + // Register a queue with the model. The model will immediately begin + // asynchronous processing of the queue (but by default, the model need not + // provide forward progress guarantees between multiple queues). + hsakmt_model_queue_t *(*register_queue)(hsakmt_model_t *model, struct hsakmt_model_queue_info *info); + + // Register a callback that allows the model to set an event. + void (*set_set_event)(hsakmt_model_t *model, hsakmt_model_set_event_fn fn, void *data); + + // Destroy a queue that was returned by register_queue. + void (*destroy_queue)(hsakmt_model_t *model, hsakmt_model_queue_t *queue); +}; + +// Type of a shared library export called `get_hsakmt_model_functions`. +typedef const struct hsakmt_model_functions *(*get_hsakmt_model_functions_t)(void); + +#endif // _HSAKMTMODELIFACE_H_ \ No newline at end of file diff --git a/projects/rocr-runtime/libhsakmt/src/debug.c b/projects/rocr-runtime/libhsakmt/src/debug.c index c6a318fb6b..6aad5ea183 100644 --- a/projects/rocr-runtime/libhsakmt/src/debug.c +++ b/projects/rocr-runtime/libhsakmt/src/debug.c @@ -284,7 +284,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCheckRuntimeDebugSupport(void) { return HSAKMT_STATUS_ERROR; //ignore cpu node - if (node.NumCPUCores) + if (node.NumCPUCores && !node.NumFComputeCores) continue; if (!node.Capability.ui32.DebugSupportedFirmware) return HSAKMT_STATUS_NOT_SUPPORTED; diff --git a/projects/rocr-runtime/libhsakmt/src/events.c b/projects/rocr-runtime/libhsakmt/src/events.c index 335174fbe7..9ab7818a4d 100644 --- a/projects/rocr-runtime/libhsakmt/src/events.c +++ b/projects/rocr-runtime/libhsakmt/src/events.c @@ -33,6 +33,7 @@ #include #include "hsakmt/linux/kfd_ioctl.h" #include "fmm.h" +#include "hsakmt/hsakmtmodel.h" static HSAuint64 *events_page = NULL; @@ -82,7 +83,10 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateEvent(HsaEventDescriptor *EventDesc, pthread_mutex_unlock(&hsakmt_mutex); return HSAKMT_STATUS_ERROR; } - hsakmt_fmm_get_handle(events_page, (uint64_t *)&args.event_page_offset); + if (hsakmt_use_model) + model_set_event_page(events_page, KFD_SIGNAL_EVENT_LIMIT); + else + hsakmt_fmm_get_handle(events_page, (uint64_t *)&args.event_page_offset); } if (hsakmt_ioctl(hsakmt_kfd_fd, AMDKFD_IOC_CREATE_EVENT, &args) != 0) { diff --git a/projects/rocr-runtime/libhsakmt/src/fmm.c b/projects/rocr-runtime/libhsakmt/src/fmm.c index 6341501e73..874d12496b 100644 --- a/projects/rocr-runtime/libhsakmt/src/fmm.c +++ b/projects/rocr-runtime/libhsakmt/src/fmm.c @@ -25,6 +25,7 @@ #include "libhsakmt.h" #include "fmm.h" +#include "hsakmt/hsakmtmodel.h" #include "hsakmt/linux/kfd_ioctl.h" #include #include @@ -2088,6 +2089,11 @@ int hsakmt_open_drm_render_device(int minor) uint32_t major_drm, minor_drm; struct amdgpu_device **device_handle; + /* Bypass amdgpu if we're running a model. Return hsakmt_kfd_fd, which is the + * backing for all our "GPU" memory. */ + if (hsakmt_use_model) + return hsakmt_kfd_fd; + if (minor < DRM_FIRST_RENDER_NODE || minor > DRM_LAST_RENDER_NODE) { pr_err("DRM render minor %d out of range [%d, %d]\n", minor, DRM_FIRST_RENDER_NODE, DRM_LAST_RENDER_NODE); @@ -2407,6 +2413,11 @@ static void *map_mmio(uint32_t node_id, uint32_t gpu_id, int mmap_fd) vm_obj->node_id = node_id; pthread_mutex_unlock(&aperture->fmm_mutex); + if (hsakmt_use_model) { + model_set_mmio_page(mem); + return mem; + } + /* Map for CPU access*/ ret = mmap(mem, PAGE_SIZE, PROT_READ | PROT_WRITE, @@ -2448,6 +2459,11 @@ HSAKMT_STATUS hsakmt_fmm_get_amdgpu_device_handle(uint32_t node_id, if (i < 0) return HSAKMT_STATUS_INVALID_NODE_UNIT; + if (hsakmt_use_model) { + *DeviceHandle = NULL; + return HSAKMT_STATUS_SUCCESS; + } + index = gpu_mem[i].drm_render_minor - DRM_FIRST_RENDER_NODE; if (!amdgpu_handle[index]) return HSAKMT_STATUS_INVALID_HANDLE; @@ -2540,6 +2556,8 @@ HSAKMT_STATUS hsakmt_fmm_init_process_apertures(unsigned int NumNodes) pagedUserptr = getenv("HSA_USERPTR_FOR_PAGED_MEM"); svm.userptr_for_paged_mem = (!pagedUserptr || strcmp(pagedUserptr, "0")); + if (hsakmt_use_model) + svm.userptr_for_paged_mem = false; /* If HSA_CHECK_USERPTR is set to a non-0 value, check all userptrs * when they are registered */ diff --git a/projects/rocr-runtime/libhsakmt/src/hsakmtmodel.c b/projects/rocr-runtime/libhsakmt/src/hsakmtmodel.c new file mode 100644 index 0000000000..aed62d03bd --- /dev/null +++ b/projects/rocr-runtime/libhsakmt/src/hsakmtmodel.c @@ -0,0 +1,823 @@ +/* + * Copyright © 2025 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "hsakmt/hsakmtmodel.h" +#include "libhsakmt.h" +#include "hsakmt/hsakmttypes.h" +#include "hsakmt/hsakmtmodeliface.h" +#define _GNU_SOURCE +#define __USE_GNU +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +bool hsakmt_use_model; +char *hsakmt_model_topology; + +struct model_node +{ + bool is_gpu; + void *aperture; + hsakmt_model_t *model; + uint64_t doorbell_offset; + uint64_t total_memory_size; + uint64_t allocated_memory_size; +}; + +struct model_event +{ + uint32_t event_type; + uint32_t auto_reset; + uint64_t value; +}; + +struct model_mem_data +{ + uint64_t va_addr; + uint64_t file_offset; + uint64_t size; + uint64_t mapped_nodes_bitmask; + uint32_t flags; + uint32_t node_id; +}; + +struct model_queue +{ + hsakmt_model_queue_t *queue; + uint32_t node_id; +}; + +#define MAX_MODEL_QUEUES 128 +// Use a 256GB aperture for the model. +#define MODEL_APERTURE_SIZE (1llu << 38) +static void *model_mmio_page; +static pthread_mutex_t model_ioctl_mutex = PTHREAD_MUTEX_INITIALIZER; +static unsigned model_event_limit; +static uint64_t *model_event_bitmap; +static struct model_event *model_events; +static pthread_cond_t model_event_condvar; +static void *model_library; +static const struct hsakmt_model_functions *model_functions; +static uint64_t model_memfd_size; +static uint64_t model_num_nodes; +static struct model_node *model_nodes; +static struct model_queue model_queues[MAX_MODEL_QUEUES]; + +HSAKMT_STATUS HSAKMTAPI hsaKmtModelEnabled(bool* enable) +{ + *enable = hsakmt_use_model; + return HSAKMT_STATUS_SUCCESS; +} + +void model_init_env_vars() +{ + /* Check whether to use a model instead of real hardware */ + hsakmt_model_topology = getenv("HSA_MODEL_TOPOLOGY"); + if (hsakmt_model_topology) + hsakmt_use_model = true; + if (hsakmt_use_model) + { + /* Backing memory file is used to stand in for the kfd_fd, + * which is needed early, so create it already. + * + * For old systems without memfd_create, or if the user prefers, + * we create a regular backing file. Prefer to use memfd_create + * by default where possible. + */ + int fd = -1; + const char *fname = getenv("HSA_MODEL_MEMFILE"); + if (fname) + { + fprintf(stderr, "model: use memory backing file given in HSA_MODEL_MEMFILE: %s\n", fname); + + fd = open(fname, O_CREAT | O_EXCL | O_CLOEXEC | O_RDWR, S_IRUSR | S_IWUSR); + if (fd < 0) + { + perror("model: failed to create backing file"); + abort(); + } + + unlink(fname); + } + + if (fd < 0) + { +#ifdef HAVE_MEMFD_CREATE + fd = memfd_create("hsakmt_model", MFD_CLOEXEC); + if (fd < 0) + { + fprintf(stderr, "model: Failed to create memfd\n"); + abort(); + } +#else + fprintf(stderr, "model: built without memfd support\n" + "model: set HSA_MODEL_MEMFILE to path of a backing file\n"); + abort(); +#endif + } + assert(hsakmt_kfd_fd < 0); + hsakmt_kfd_fd = fd; + pthread_condattr_t condattr; + pthread_condattr_init(&condattr); + pthread_condattr_setclock(&condattr, CLOCK_MONOTONIC); + pthread_cond_init(&model_event_condvar, &condattr); + pthread_condattr_destroy(&condattr); + const char *libname = getenv("HSA_MODEL_LIB"); + if (!libname) + { + fprintf(stderr, "model: HSA_MODEL_LIB environment variable must be set to FFM .so\n"); + abort(); + } + // model_library = dlmopen(LM_ID_NEWLM, libname, RTLD_NOW); + model_library = dlopen(libname, RTLD_NOW | RTLD_LOCAL); + if (!model_library) + { + fprintf(stderr, "model: failed to load %s: %s\n", libname, dlerror()); + abort(); + } + get_hsakmt_model_functions_t getter = dlsym(model_library, "get_hsakmt_model_functions"); + if (!getter) + { + fprintf(stderr, "model: Failed to get hsakmt_model_functions\n"); + abort(); + } + model_functions = getter(); + if (model_functions->version_major != HSAKMT_MODEL_INTERFACE_VERSION_MAJOR || + model_functions->version_minor < HSAKMT_MODEL_INTERFACE_VERSION_MINOR) + { + fprintf(stderr, "model: Model has interface version %u.%u, need version %u.%u\n", + model_functions->version_major, model_functions->version_minor, + HSAKMT_MODEL_INTERFACE_VERSION_MAJOR, HSAKMT_MODEL_INTERFACE_VERSION_MINOR); + abort(); + } + } +} + +static uint64_t allocate_from_memfd(uint64_t size, uint64_t align) +{ + if (!align) + align = 4096; + assert(POWER_OF_2(align)); /* must be power of two */ + assert(align >= 4096); + size = (size + 4095) & ~4095; + model_memfd_size = (model_memfd_size + align - 1) & ~(align - 1); + uint64_t offset = model_memfd_size; + model_memfd_size += size; + int ret = ftruncate(hsakmt_kfd_fd, model_memfd_size); + if (ret < 0) + { + fprintf(stderr, "model: ftruncate on memfd failed\n"); + abort(); + } + return offset; +} +static uint64_t get_sysfs_mem_bank_size(unsigned node_id, unsigned mem_id) +{ + char prop_name[256]; + char path[256]; + snprintf(path, sizeof(path), "%s/nodes/%u/mem_banks/%u/properties", + hsakmt_model_topology, node_id, mem_id); + FILE *f = fopen(path, "r"); + if (!f) + { + fprintf(stderr, "model: Failed to open %s\n", path); + abort(); + } + uint64_t prop_val; + while (fscanf(f, "%s %" PRIu64 "\n", prop_name, &prop_val) == 2) + { + if (!strcmp(prop_name, "size_in_bytes")) + { + fclose(f); + return prop_val; + } + } + fprintf(stderr, "model: Missing size_in_bytes in %s\n", path); + abort(); +} + +static void model_set_event(void *data, unsigned event_id) +{ + if (!event_id) + return; + + if (event_id > model_event_limit) + { + fprintf(stderr, "model_set_event: event_id = %u out of bounds\n", + event_id); + abort(); + } + + unsigned slot = event_id - 1; + + if (!((model_event_bitmap[slot / 64] >> (slot % 64)) & 1)) + { + fprintf(stderr, "model_set_event: event_id = %u is not allocated\n", + event_id); + abort(); + } + + struct model_event *event = &model_events[slot]; + if (event->event_type == HSA_EVENTTYPE_SIGNAL) + { + assert(model_events[slot].value <= 1); + model_events[slot].value = 1; + } + else + { + fprintf(stderr, "model: Unimplemented event type\n"); + abort(); + } + + pthread_cond_broadcast(&model_event_condvar); +} + +void model_init() +{ + if (!hsakmt_use_model) + return; + HSAKMT_STATUS result; + HsaSystemProperties props; + /* Read the topology to determine nodes. */ + result = hsakmt_topology_sysfs_get_system_props(&props); + if (result != HSAKMT_STATUS_SUCCESS) + { + fprintf(stderr, "model: Failed to parse topology\n"); + abort(); + } + model_nodes = calloc(props.NumNodes, sizeof(*model_nodes)); + if (!model_nodes) + abort(); + model_num_nodes = props.NumNodes; + for (unsigned node_id = 0; node_id < props.NumNodes; node_id++) + { + HsaNodeProperties node_props; + result = hsakmt_topology_get_node_props(node_id, &node_props); + if (result != HSAKMT_STATUS_SUCCESS) + { + fprintf(stderr, "model: Failed to get node %u properties\n", node_id); + abort(); + } + if (node_props.KFDGpuID == 0) + continue; + if (node_props.KFDGpuID != node_id + 1) + { + fprintf(stderr, + "model: Node %u has KFD GPU ID %u, but should be %u." + " Please change the gpu_id file.\n", + node_id, node_props.KFDGpuID, node_id + 1); + abort(); + } + model_nodes[node_id].is_gpu = true; + /* Reserve the VA space for the aperture, but don't fill it with pages. */ + model_nodes[node_id].aperture = + mmap(NULL, MODEL_APERTURE_SIZE, PROT_NONE, + MAP_PRIVATE | MAP_NORESERVE | MAP_ANONYMOUS, -1, 0); + pr_debug("Modeling Creating Memory Aperture: %p\n", model_nodes[node_id].aperture); + if (model_nodes[node_id].aperture == MAP_FAILED) + { + fprintf(stderr, "model: Failed to reserve aperture via mmap\n"); + abort(); + } + /* Create the doorbell region */ + model_nodes[node_id].doorbell_offset = allocate_from_memfd(8192, 8192); + for (unsigned mem_id = 0; mem_id < node_props.NumMemoryBanks; ++mem_id) + { + model_nodes[node_id].total_memory_size += get_sysfs_mem_bank_size(node_id, mem_id); + } + /* Create the model */ + // TODO: Move this into a separate thread + model_nodes[node_id].model = model_functions->create(); + if (!model_nodes[node_id].model) + { + fprintf(stderr, "model: Failed to create model\n"); + abort(); + } + model_functions->set_global_aperture(model_nodes[node_id].model, + model_nodes[node_id].aperture, + MODEL_APERTURE_SIZE); + + model_functions->set_set_event(model_nodes[node_id].model, model_set_event, NULL); + } +} +void model_set_mmio_page(void *ptr) +{ + assert(!model_mmio_page); + model_mmio_page = ptr; +} +void model_set_event_page(void *ptr, unsigned event_limit) +{ + // TODO: Fully understand what's happening with this page and the event limit. + // ROCR-Runtime allocates a pool of 4096 events, but also a handful or so + // of additional events, which blows through the event_limit of 4096 + // that is passed here. And it seems that not using the page at all + // is supported? + assert(!model_event_limit); + assert(event_limit % 64 == 0); + event_limit *= 2; + model_event_limit = event_limit; + model_event_bitmap = calloc(event_limit / 64, 8); + model_events = calloc(event_limit, sizeof(*model_events)); +} +/* Model implementation of KFD ioctl. */ + +static int model_kfd_ioctl_locked(unsigned long request, void *arg) +{ + assert(_IOC_TYPE(request) == AMDKFD_IOCTL_BASE); + if (_IOC_NR(request) == 0x20) + { + // This is AMDKFD_IOC_SVM. It is defined / used in an unusual way. + struct kfd_ioctl_svm_args *args = arg; + if (args->op == KFD_IOCTL_SVM_OP_SET_ATTR) + { + // todo? + return 0; + } + fprintf(stderr, "model: Unimplemented SVM op\n"); + abort(); + } + switch (request) + { + case AMDKFD_IOC_GET_VERSION: + { + pr_debug("MODEL IOCTL: AMDKFD_IOC_GET_VERSION\n"); + struct kfd_ioctl_get_version_args *args = arg; + args->major_version = 1; + args->minor_version = 14; + return 0; + } + case AMDKFD_IOC_GET_PROCESS_APERTURES_NEW: + { + pr_debug("MODEL IOCTL: AMDKFD_IOC_GET_PROCESS_APERTURES_NEW\n"); + struct kfd_ioctl_get_process_apertures_new_args *args = arg; + struct kfd_process_device_apertures *apertures = + (void *)args->kfd_process_device_apertures_ptr; + assert(args->num_of_nodes == model_num_nodes); + for (unsigned node_id = 0; node_id < args->num_of_nodes; ++node_id) + { + memset(&apertures[node_id], 0, sizeof(apertures[node_id])); + if (!model_nodes[node_id].is_gpu) + continue; + apertures[node_id].gpu_id = 1 + node_id; + apertures[node_id].gpuvm_base = 0x4000llu; + apertures[node_id].gpuvm_limit = MODEL_APERTURE_SIZE; + apertures[node_id].lds_base = 0x4000000000000000llu; // 0x1000000000000? + apertures[node_id].lds_limit = 0x40000000ffffffffllu; + apertures[node_id].scratch_base = 0x5000000000000000llu; // 0x2000000000000? + apertures[node_id].scratch_limit = 0x50000000ffffffffllu; + } + return 0; + } + case AMDKFD_IOC_SET_XNACK_MODE: + { + pr_debug("MODEL IOCTL: AMDKFD_IOC_SET_XNACK_MODE\n"); + // Don't support XNACK + struct kfd_ioctl_set_xnack_mode_args *args = arg; + if (args->xnack_enabled < 0) + { + args->xnack_enabled = 0; + return 0; + } + errno = EPERM; + return -1; + } + case AMDKFD_IOC_GET_CLOCK_COUNTERS: + { + pr_debug("MODEL IOCTL: AMDKFD_IOC_GET_CLOCK_COUNTERS\n"); + struct kfd_ioctl_get_clock_counters_args *args = arg; + args->gpu_clock_counter = 0; // TODO + args->cpu_clock_counter = 0; + args->system_clock_counter = 0; + args->system_clock_freq = 0; + return 0; + } + case AMDKFD_IOC_ACQUIRE_VM: + pr_debug("MODEL IOCTL: AMDKFD_IOC_ACQUIRE_VM\n"); + return 0; + case AMDKFD_IOC_SET_MEMORY_POLICY: + { + pr_debug("MODEL IOCTL: AMDKFD_IOC_SET_MEMORY_POLICY\n"); + // todo? + return 0; + } + case AMDKFD_IOC_AVAILABLE_MEMORY: + { + pr_debug("MODEL IOCTL: AMDKFD_IOC_AVAILABLE_MEMORY\n"); + static const uint64_t minimum_reported = 128 * 1024 * 1024; + struct kfd_ioctl_get_available_memory_args *args = arg; + unsigned node_id = args->gpu_id - 1; + struct model_node *node = &model_nodes[node_id]; + assert(node_id < model_num_nodes); + if (node->allocated_memory_size + minimum_reported >= node->total_memory_size) + args->available = minimum_reported; + else + args->available = node->total_memory_size - node->allocated_memory_size; + return 0; + } + case AMDKFD_IOC_ALLOC_MEMORY_OF_GPU: + { + // Expect an SVM style allocation: The memory is allocated on the host + // side e.g. via mmap(), and this IOCTL "only" registers the memory + // with the GPU. This is a no-op for us because we aren't a GPU. + struct kfd_ioctl_alloc_memory_of_gpu_args *args = arg; + unsigned node_id = args->gpu_id - 1; + assert(node_id < model_num_nodes); + assert(model_nodes[node_id].is_gpu); + if (args->va_addr == 0) + { + fprintf(stderr, "model: Expect only SVM allocations?\n"); + abort(); + } + if (args->size % PAGE_SIZE != 0) + { + fprintf(stderr, "model: Allocation size not a multiple of page size\n"); + abort(); + } + if (args->flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) + { + fprintf(stderr, "model: userptr not supported\n"); + abort(); + } + struct model_mem_data *mem_data = calloc(1, sizeof(*mem_data)); + if (!mem_data) + abort(); + mem_data->va_addr = args->va_addr; + mem_data->size = args->size; + mem_data->flags = args->flags; + mem_data->node_id = node_id; + if (args->flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) + { + assert(args->size == 8192); + mem_data->file_offset = model_nodes[node_id].doorbell_offset; + } + else + { + mem_data->file_offset = allocate_from_memfd(args->size, 0); + } + args->handle = (__u64)mem_data; + args->mmap_offset = mem_data->file_offset; + model_nodes[node_id].allocated_memory_size += args->size; + pr_debug("MODEL IOCTL: AMDKFD_IOC_ALLOC_MEMORY_OF_GPU: VA: %lx : Size: %lu, Flags: %x\n", mem_data->va_addr, mem_data->size, mem_data->flags); + model_functions->alloced_memory(model_nodes[node_id].model, (uint64_t *)mem_data->va_addr, mem_data->size, mem_data->flags); + return 0; + } + case AMDKFD_IOC_FREE_MEMORY_OF_GPU: + { + struct kfd_ioctl_free_memory_of_gpu_args *args = arg; + struct model_mem_data *mem_data = (void *)args->handle; + assert(!mem_data->mapped_nodes_bitmask); + // Free the memory by punching a hole into the underlying memfd. + // + // Ideally, we'd also remember holes in the file and re-use them for + // allocations to avoid the file size from growing indefinitely. It's + // unclear whether the current implementation causes kernel data + // structures to grow. But in practice, it almost certainly never + // matters. + int ret = fallocate(hsakmt_kfd_fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + mem_data->file_offset, mem_data->size); + if (ret != 0) + { + perror("model: failed to punch hole in memfd"); + abort(); + } + model_nodes[mem_data->node_id].allocated_memory_size -= mem_data->size; + model_functions->freed_memory(model_nodes[mem_data->node_id].model, (uint64_t *)mem_data->va_addr, mem_data->size); + pr_debug("MODEL IOCTL: AMDKFD_IOC_FREE_MEMORY_OF_GPU: VA: %lx : Size: %lu, Flags: %x\n", mem_data->va_addr, mem_data->size, mem_data->flags); + free(mem_data); + return 0; + } + case AMDKFD_IOC_MAP_MEMORY_TO_GPU: + { + struct kfd_ioctl_map_memory_to_gpu_args *args = arg; + struct model_mem_data *mem_data = (void *)args->handle; + while (args->n_success < args->n_devices) + { + uint32_t gpu_id = ((uint32_t *)args->device_ids_array_ptr)[args->n_success]; + uint32_t node_id = gpu_id - 1; + assert(node_id < model_num_nodes); + if (mem_data->mapped_nodes_bitmask & (1llu << node_id)) + { + fprintf(stderr, "model: Already mapped\n"); + abort(); + } + assert(model_nodes[node_id].aperture); + unsigned prot = PROT_READ; + if (mem_data->flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE) + prot |= PROT_WRITE; + // TODO: Mark *shader*-executable memory? + + pr_debug("MODEL IOCTL: AMDKFD_IOC_MAP_MEMORY_TO_GPU: VA: %lx : Size: %lu, Flags: %x\n", mem_data->va_addr, mem_data->size, mem_data->flags); + void *ret = mmap(VOID_PTR_ADD(model_nodes[node_id].aperture, mem_data->va_addr), + mem_data->size, prot, + MAP_SHARED | MAP_FIXED, hsakmt_kfd_fd, mem_data->file_offset); + if (ret == MAP_FAILED) + { + fprintf(stderr, "model: mmap failed\n"); + abort(); + } + mem_data->mapped_nodes_bitmask |= (1llu << node_id); + args->n_success++; + } + return 0; + } + case AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU: + { + pr_debug("MODEL IOCTL: AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU\n"); + struct kfd_ioctl_unmap_memory_from_gpu_args *args = arg; + struct model_mem_data *mem_data = (void *)args->handle; + while (args->n_success < args->n_devices) + { + uint32_t gpu_id = ((uint32_t *)args->device_ids_array_ptr)[args->n_success]; + uint32_t node_id = gpu_id - 1; + assert(node_id < model_num_nodes); + if (!(mem_data->mapped_nodes_bitmask & (1llu << node_id))) + { + fprintf(stderr, "model: Not mapped\n"); + abort(); + } + assert(model_nodes[node_id].aperture); + /* Overwrite the mapping with an empty mapping to keep + * it reserved. */ + void *ret = mmap(VOID_PTR_ADD(model_nodes[node_id].aperture, mem_data->va_addr), + mem_data->size, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED | MAP_NORESERVE, -1, 0); + if (ret == MAP_FAILED) + { + perror("model: unmap failed"); + abort(); + } + mem_data->mapped_nodes_bitmask &= ~(1llu << node_id); + args->n_success++; + } + args->n_success = args->n_devices; + return 0; + } + case AMDKFD_IOC_CREATE_EVENT: + { + struct kfd_ioctl_create_event_args *args = arg; + pr_debug("MODEL IOCTL: AMDKFD_IOC_CREATE_EVENT: %u\n", args->event_type); + // Find a free slot + unsigned i; + for (i = 0; i < model_event_limit; i += 64) + { + uint64_t bitmap = model_event_bitmap[i / 64]; + if (bitmap == ~(uint64_t)0) + continue; + i += ffsll(~bitmap) - 1; + break; + } + if (i >= model_event_limit) + { + fprintf(stderr, "model: Ran out of event slots. Should be an application error.\n"); + abort(); + } + // Allocate the signal + model_event_bitmap[i / 64] |= (uint64_t)1 << (i % 64); + model_events[i].event_type = args->event_type; + model_events[i].auto_reset = args->auto_reset; + model_events[i].value = 0; + args->event_trigger_data = 0xbadf001; // ??? + args->event_id = 1 + i; + args->event_slot_index = ~0; + return 0; + } + case AMDKFD_IOC_WAIT_EVENTS: + { + struct kfd_ioctl_wait_events_args *args = arg; + struct kfd_event_data *events = (void *)args->events_ptr; + pr_debug("MODEL IOCTL: AMDKFD_IOC_WAIT_EVENTS: %u\n", args->num_events); + bool have_timeout = args->timeout != 0xffffffffu; + bool hit_timeout = false; + struct timespec timeout; + if (have_timeout) + { + clock_gettime(CLOCK_MONOTONIC, &timeout); + timeout.tv_sec += args->timeout / 1000; + timeout.tv_nsec += (args->timeout % 1000) * 1000000; + if (timeout.tv_nsec > 1000000000) + { + timeout.tv_nsec -= 1000000000; + timeout.tv_sec++; + } + } + for (;;) + { + bool final_ready = args->wait_for_all; + for (unsigned i = 0; i < args->num_events; ++i) + { + unsigned slot = events[i].event_id - 1; + struct model_event *event = &model_events[slot]; + bool this_ready; + if (event->event_type == HSA_EVENTTYPE_SIGNAL) + { + uint64_t current_age = event->value; + uint64_t target_age = events[i].signal_event_data.last_event_age; + this_ready = current_age >= target_age; + } + else if (event->event_type == HSA_EVENTTYPE_HW_EXCEPTION || + event->event_type == HSA_EVENTTYPE_NODECHANGE || + event->event_type == HSA_EVENTTYPE_DEVICESTATECHANGE || + event->event_type == HSA_EVENTTYPE_HW_EXCEPTION || + event->event_type == HSA_EVENTTYPE_DEBUG_EVENT || + event->event_type == HSA_EVENTTYPE_PROFILE_EVENT || + event->event_type == HSA_EVENTTYPE_MEMORY) + { + // These never happen in the model + } + else + { + fprintf(stderr, "model: Unimplemented event type\n"); + abort(); + } + if (final_ready != this_ready) + { + final_ready = this_ready; + break; + } + } + if (final_ready) + break; + if (have_timeout) + { + int ret = pthread_cond_timedwait( + &model_event_condvar, &model_ioctl_mutex, &timeout); + if (ret == ETIMEDOUT) + { + hit_timeout = true; + break; + } + } + else + { + pthread_cond_wait(&model_event_condvar, &model_ioctl_mutex); + } + } + /* Record most recent event ages and perform auto reset. */ + for (unsigned i = 0; i < args->num_events; ++i) + { + unsigned slot = events[i].event_id - 1; + struct model_event *event = &model_events[slot]; + if (event->event_type == HSA_EVENTTYPE_SIGNAL) + { + uint64_t last_age = event->value; + if (event->auto_reset && last_age >= events[i].signal_event_data.last_event_age) + event->value = 0; + events[i].signal_event_data.last_event_age = last_age; + } + } + args->wait_result = hit_timeout ? KFD_IOC_WAIT_RESULT_TIMEOUT + : KFD_IOC_WAIT_RESULT_COMPLETE; + return 0; + } + case AMDKFD_IOC_SET_EVENT: + { + struct kfd_ioctl_set_event_args *args = arg; + model_set_event(NULL, args->event_id); + return 0; + } + case AMDKFD_IOC_RESET_EVENT: + { + pr_debug("MODEL IOCTL: AMDKFD_IOC_RESET_EVENT\n"); + struct kfd_ioctl_reset_event_args *args = arg; + unsigned slot = args->event_id - 1; + struct model_event *event = &model_events[slot]; + if (event->event_type == HSA_EVENTTYPE_SIGNAL) + { + model_events[slot].value = 0; + } + else + { + fprintf(stderr, "model: Unimplemented event type\n"); + abort(); + } + return 0; + } + case AMDKFD_IOC_DESTROY_EVENT: + { + struct kfd_ioctl_destroy_event_args *args = arg; + unsigned i = args->event_id - 1; + if (i >= model_event_limit || !(model_event_bitmap[i / 64] & ((uint64_t)1 << (i % 64)))) + { + fprintf(stderr, "model: trying to destroy an event that doesn't exist.\n"); + abort(); + } + memset(&model_events[i], 0, sizeof(model_events[i])); + model_event_bitmap[i / 64] &= ~((uint64_t)1 << (i % 64)); + return 0; + } + case AMDKFD_IOC_CREATE_QUEUE: + { + pr_debug("MODEL IOCTL: AMDKFD_IOC_CREATE_QUEUE\n"); + struct kfd_ioctl_create_queue_args *args = arg; + unsigned node_id = args->gpu_id - 1; + assert(node_id < model_num_nodes); + assert(model_nodes[node_id].model); + const bool supported_queue_type = args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL || + args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA; + if (!supported_queue_type) + { + fprintf(stderr, "model: Unsupported queue type\n"); + abort(); + } + unsigned queue_id = 0; + while (queue_id < MAX_MODEL_QUEUES && model_queues[queue_id].queue) + queue_id++; + if (queue_id >= MAX_MODEL_QUEUES) + { + fprintf(stderr, "model: too many queues\n"); + abort(); + } + struct hsakmt_model_queue_info info = {0}; + info.ring_base_address = args->ring_base_address; + info.ring_size = args->ring_size; + info.write_pointer_address = args->write_pointer_address; + info.read_pointer_address = args->read_pointer_address; + info.queue_type = args->queue_type; + model_queues[queue_id].queue = + model_functions->register_queue(model_nodes[node_id].model, &info); + model_queues[queue_id].node_id = node_id; + args->queue_id = queue_id; + // Note that strictly speaking, this is the offset into the hsakmt_kfd_fd + // file, not the DRM fd (but they are the same in our case). + args->doorbell_offset = model_nodes[node_id].doorbell_offset + 8 * queue_id; + return 0; + } + case AMDKFD_IOC_DESTROY_QUEUE: + { + struct kfd_ioctl_destroy_queue_args *args = arg; + if (args->queue_id >= MAX_MODEL_QUEUES || !model_queues[args->queue_id].queue) + { + fprintf(stderr, "model: trying to destroy a queue that doesn't exist\n"); + abort(); + } + struct model_queue *queue = &model_queues[args->queue_id]; + // Older model versions simply leak the queue. + if (model_functions->version_minor >= 3) + model_functions->destroy_queue(model_nodes[queue->node_id].model, queue->queue); + queue->queue = NULL; + return 0; + } + case AMDKFD_IOC_GET_TILE_CONFIG: + { + pr_debug("MODEL IOCTL: AMDKFD_IOC_GET_TILE_CONFIG\n"); + struct kfd_ioctl_get_tile_config_args *args = arg; + args->gb_addr_config = 0x10000444; + return 0; + } + case AMDKFD_IOC_SET_SCRATCH_BACKING_VA: + pr_debug("MODEL IOCTL: AMDKFD_IOC_SET_SCRATCH_BACKING_VA\n"); + // no-op -- scratch allocations are communicated via amd_queue_s + return 0; + case AMDKFD_IOC_RUNTIME_ENABLE: + pr_debug("MODEL IOCTL: AMDKFD_IOC_RUNTIME_ENABLE\n"); + fprintf(stderr, "model: Debugger runtime not implemented\n"); + fprintf(stderr, "Fix this by clearing bit 30 of the 'capability' field in $HSA_MODEL_TOPOLOGY/%%d/properties\n"); + abort(); + default: + fprintf(stderr, "model: Unimplemented KFD ioctl\n"); + abort(); + } +} +int model_kfd_ioctl(unsigned long request, void *arg) +{ + /* Use a very simle locking strategy for correctness. IOCTLs should + * be rare anyway and not contended considering the cost of running + * the model itself. + * + * The bulk of model execution happens in a separate thread *without* + * holding the IOCTL mutex. */ + pthread_mutex_lock(&model_ioctl_mutex); + int ret = model_kfd_ioctl_locked(request, arg); + pthread_mutex_unlock(&model_ioctl_mutex); + return ret; +} \ No newline at end of file diff --git a/projects/rocr-runtime/libhsakmt/src/libhsakmt.c b/projects/rocr-runtime/libhsakmt/src/libhsakmt.c index d1021aa7e6..2f2f2990fc 100644 --- a/projects/rocr-runtime/libhsakmt/src/libhsakmt.c +++ b/projects/rocr-runtime/libhsakmt/src/libhsakmt.c @@ -3,10 +3,14 @@ #include #include "libhsakmt.h" +#include "hsakmt/hsakmtmodel.h" /* Call ioctl, restarting if it is interrupted */ int hsakmt_ioctl(int fd, unsigned long request, void *arg) { + if (hsakmt_use_model) + return model_kfd_ioctl(request, arg); + int ret; do { diff --git a/projects/rocr-runtime/libhsakmt/src/libhsakmt.h b/projects/rocr-runtime/libhsakmt/src/libhsakmt.h index 329cbd39d0..245a621f40 100644 --- a/projects/rocr-runtime/libhsakmt/src/libhsakmt.h +++ b/projects/rocr-runtime/libhsakmt/src/libhsakmt.h @@ -236,6 +236,8 @@ extern int hsakmt_ioctl(int fd, unsigned long request, void *arg); typeof(a) tmp1 = (a), tmp2 = (b); \ tmp1 > tmp2 ? tmp1 : tmp2; }) +#define POWER_OF_2(x) ((x && (!(x & (x - 1)))) ? 1 : 0) + void hsakmt_clear_events_page(void); void hsakmt_fmm_clear_all_mem(void); void hsakmt_clear_process_doorbells(void); diff --git a/projects/rocr-runtime/libhsakmt/src/memory.c b/projects/rocr-runtime/libhsakmt/src/memory.c index ba2a2175bb..6d36b77d59 100644 --- a/projects/rocr-runtime/libhsakmt/src/memory.c +++ b/projects/rocr-runtime/libhsakmt/src/memory.c @@ -112,8 +112,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemory(HSAuint32 PreferredNode, return hsaKmtAllocMemoryAlign(PreferredNode, SizeInBytes, 0, MemFlags, MemoryAddress); } -#define POWER_OF_2(x) ((x && (!(x & (x - 1)))) ? 1 : 0) - HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemoryAlign(HSAuint32 PreferredNode, HSAuint64 SizeInBytes, HSAuint64 Alignment, diff --git a/projects/rocr-runtime/libhsakmt/src/openclose.c b/projects/rocr-runtime/libhsakmt/src/openclose.c index a02804893f..4b0b21019b 100644 --- a/projects/rocr-runtime/libhsakmt/src/openclose.c +++ b/projects/rocr-runtime/libhsakmt/src/openclose.c @@ -29,6 +29,7 @@ #define _GNU_SOURCE #include "libhsakmt.h" +#include "hsakmt/hsakmtmodel.h" #include #include @@ -174,7 +175,10 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) if (result != HSAKMT_STATUS_SUCCESS) goto open_failed; - if (hsakmt_kfd_fd < 0) { + // Check if we are using the hsakmtmodel and setup initial state + model_init_env_vars(); + + if (hsakmt_kfd_fd < 0 && !hsakmt_use_model) { fd = open(kfd_device_name, O_RDWR | O_CLOEXEC); if (fd == -1) { @@ -193,8 +197,9 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) useSvmStr = getenv("HSA_USE_SVM"); hsakmt_is_svm_api_supported = !(useSvmStr && !strcmp(useSvmStr, "0")); - - result = hsakmt_topology_sysfs_get_system_props(&sys_props); + if(!hsakmt_use_model) + result = hsakmt_topology_sysfs_get_system_props(&sys_props); + if (result != HSAKMT_STATUS_SUCCESS) goto topology_sysfs_failed; diff --git a/projects/rocr-runtime/libhsakmt/src/topology.c b/projects/rocr-runtime/libhsakmt/src/topology.c index 6ae2106ca3..574c3ecb27 100644 --- a/projects/rocr-runtime/libhsakmt/src/topology.c +++ b/projects/rocr-runtime/libhsakmt/src/topology.c @@ -41,6 +41,7 @@ #include #include "libhsakmt.h" +#include "hsakmt/hsakmtmodel.h" #include "fmm.h" /* Number of memory banks added by thunk on top of topology @@ -52,9 +53,17 @@ #define NUM_OF_IGPU_HEAPS 3 #define NUM_OF_DGPU_HEAPS 3 /* SYSFS related */ -#define KFD_SYSFS_PATH_GENERATION_ID "/sys/devices/virtual/kfd/kfd/topology/generation_id" -#define KFD_SYSFS_PATH_SYSTEM_PROPERTIES "/sys/devices/virtual/kfd/kfd/topology/system_properties" -#define KFD_SYSFS_PATH_NODES "/sys/devices/virtual/kfd/kfd/topology/nodes" +#define KFD_SYSFS_PATH "/sys/devices/virtual/kfd/kfd/topology" +#define KFD_SYSFS_PATH_GENERATION_ID "%s/generation_id" +#define KFD_SYSFS_PATH_SYSTEM_PROPERTIES "%s/system_properties" +#define KFD_SYSFS_PATH_NODES "%s/nodes" + +static const char *get_topology_dir(void) +{ + if (hsakmt_use_model) + return hsakmt_model_topology; + return KFD_SYSFS_PATH; +} typedef struct { HsaNodeProperties node; @@ -584,8 +593,11 @@ static HSAKMT_STATUS topology_sysfs_get_generation(uint32_t *gen) FILE *fd; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + char path[256]; + snprintf(path, sizeof(path), KFD_SYSFS_PATH_GENERATION_ID, get_topology_dir()); + assert(gen); - fd = fopen(KFD_SYSFS_PATH_GENERATION_ID, "r"); + fd = fopen(path, "r"); if (!fd) return HSAKMT_STATUS_ERROR; if (fscanf(fd, "%ul", gen) != 1) { @@ -614,7 +626,7 @@ static HSAKMT_STATUS topology_sysfs_get_gpu_id(uint32_t sysfs_node_id, uint32_t HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; assert(gpu_id); - snprintf(path, 256, "%s/%d/gpu_id", KFD_SYSFS_PATH_NODES, sysfs_node_id); + snprintf(path, sizeof(path), KFD_SYSFS_PATH_NODES "/%d/gpu_id", get_topology_dir(), sysfs_node_id); fd = fopen(path, "r"); if (!fd) return HSAKMT_STATUS_ERROR; @@ -666,7 +678,7 @@ static HSAKMT_STATUS topology_sysfs_check_node_supported(uint32_t sysfs_node_id, return HSAKMT_STATUS_NO_MEMORY; /* Retrieve the node properties */ - snprintf(path, 256, "%s/%d/properties", KFD_SYSFS_PATH_NODES, sysfs_node_id); + snprintf(path, 256, KFD_SYSFS_PATH_NODES "/%d/properties", get_topology_dir(), sysfs_node_id); fd = fopen(path, "r"); if (!fd) { free(read_buf); @@ -715,6 +727,7 @@ HSAKMT_STATUS hsakmt_topology_sysfs_get_system_props(HsaSystemProperties *props) { FILE *fd; char *read_buf, *p; + char path[256]; char prop_name[256]; unsigned long long prop_val; uint32_t prog; @@ -724,7 +737,8 @@ HSAKMT_STATUS hsakmt_topology_sysfs_get_system_props(HsaSystemProperties *props) uint32_t num_supported_nodes = 0; assert(props); - fd = fopen(KFD_SYSFS_PATH_SYSTEM_PROPERTIES, "r"); + snprintf(path, sizeof(path), KFD_SYSFS_PATH_SYSTEM_PROPERTIES, get_topology_dir()); + fd = fopen(path, "r"); if (!fd) return HSAKMT_STATUS_ERROR; @@ -762,7 +776,8 @@ HSAKMT_STATUS hsakmt_topology_sysfs_get_system_props(HsaSystemProperties *props) * Assuming that inside nodes folder there are only folders * which represent the node numbers */ - num_sysfs_nodes = num_subdirs(KFD_SYSFS_PATH_NODES, ""); + snprintf(path, sizeof(path), KFD_SYSFS_PATH_NODES, get_topology_dir()); + num_sysfs_nodes = num_subdirs(path, ""); if (map_user_to_sysfs_node_id == NULL) { /* Trade off - num_sysfs_nodes includes all CPU and GPU nodes. @@ -1095,7 +1110,7 @@ static HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, return HSAKMT_STATUS_NO_MEMORY; /* Retrieve the node properties */ - snprintf(path, 256, "%s/%d/properties", KFD_SYSFS_PATH_NODES, sys_node_id); + snprintf(path, 256, KFD_SYSFS_PATH_NODES "/%d/properties", get_topology_dir(), sys_node_id); fd = fopen(path, "r"); if (!fd) { free(read_buf); @@ -1197,6 +1212,8 @@ static HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, props->NumCpQueues = prop_val; else if (strcmp(prop_name, "num_xcc") == 0) props->NumXcc = prop_val; + else if (strcmp(prop_name, "family_id") == 0) + props->FamilyID = prop_val; else if (strcmp(prop_name, "gfx_target_version") == 0) gfxv = (uint32_t)prop_val; } @@ -1302,7 +1319,7 @@ static HSAKMT_STATUS topology_sysfs_get_mem_props(uint32_t node_id, if (ret != HSAKMT_STATUS_SUCCESS) return ret; - snprintf(path, 256, "%s/%d/mem_banks/%d/properties", KFD_SYSFS_PATH_NODES, sys_node_id, mem_id); + snprintf(path, 256, KFD_SYSFS_PATH_NODES "/%d/mem_banks/%d/properties", get_topology_dir(), sys_node_id, mem_id); fd = fopen(path, "r"); if (!fd) return HSAKMT_STATUS_ERROR; @@ -1536,7 +1553,7 @@ static HSAKMT_STATUS topology_sysfs_get_cache_props(uint32_t node_id, if (ret != HSAKMT_STATUS_SUCCESS) return ret; - snprintf(path, 256, "%s/%d/caches/%d/properties", KFD_SYSFS_PATH_NODES, sys_node_id, cache_id); + snprintf(path, 256, KFD_SYSFS_PATH_NODES "/%d/caches/%d/properties", get_topology_dir(), sys_node_id, cache_id); fd = fopen(path, "r"); if (!fd) return HSAKMT_STATUS_ERROR; @@ -1633,10 +1650,7 @@ static HSAKMT_STATUS topology_sysfs_get_iolink_props(uint32_t node_id, if (ret != HSAKMT_STATUS_SUCCESS) return ret; - if (p2pLink) - snprintf(path, 256, "%s/%d/p2p_links/%d/properties", KFD_SYSFS_PATH_NODES, sys_node_id, iolink_id); - else - snprintf(path, 256, "%s/%d/io_links/%d/properties", KFD_SYSFS_PATH_NODES, sys_node_id, iolink_id); + snprintf(path, 256, KFD_SYSFS_PATH_NODES "/%d/%s/%d/properties", get_topology_dir(), sys_node_id, p2pLink ? "p2p_links" : "io_links", iolink_id); fd = fopen(path, "r"); if (!fd) @@ -2188,6 +2202,9 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAcquireSystemProperties(HsaSystemProperties *Syste assert(g_system); + if (hsakmt_use_model) + model_init(); + err = hsakmt_fmm_init_process_apertures(g_system->NumNodes); if (err != HSAKMT_STATUS_SUCCESS) goto init_process_apertures_failed; diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/driver/kfd/amd_kfd_driver.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/driver/kfd/amd_kfd_driver.cpp index afc88a7a19..addafc480c 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/driver/kfd/amd_kfd_driver.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/driver/kfd/amd_kfd_driver.cpp @@ -520,5 +520,14 @@ bool KfdDriver::BindXnackMode() { return (mode != Flag::XNACK_DISABLE); } +hsa_status_t KfdDriver::IsModelEnabled(bool* enable) const { + // AIE does not support streaming performance monitor. + HSAKMT_STATUS status = HSAKMT_STATUS_ERROR; + status = hsaKmtModelEnabled(enable); + if (status != HSAKMT_STATUS_SUCCESS) { + return HSA_STATUS_ERROR; + } +} + } // namespace AMD } // namespace rocr diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/driver/xdna/amd_xdna_driver.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/driver/xdna/amd_xdna_driver.cpp index 26b52cd6f3..c170c4810d 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/driver/xdna/amd_xdna_driver.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/driver/xdna/amd_xdna_driver.cpp @@ -688,5 +688,11 @@ hsa_status_t XdnaDriver::SPMSetDestBuffer(uint32_t preferred_node_id, uint32_t s return HSA_STATUS_ERROR_INVALID_AGENT; } +hsa_status_t XdnaDriver::IsModelEnabled(bool* enable) const { + // AIE does not support streaming performance monitor. + *enable = false; + return HSA_STATUS_SUCCESS; +} + } // namespace AMD } // namespace rocr diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_kfd_driver.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_kfd_driver.h index adf53e6785..4579b6e8b0 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_kfd_driver.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_kfd_driver.h @@ -115,6 +115,8 @@ public: uint32_t* size_copied, void* dest_mem_addr, bool* is_spm_data_loss) const override; + hsa_status_t IsModelEnabled(bool* enable) const override; + private: /// @brief Allocate agent accessible memory (system / local memory). static void *AllocateKfdMemory(const HsaMemFlags &flags, uint32_t node_id, diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_xdna_driver.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_xdna_driver.h index b9c4cd68aa..ef871e6960 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_xdna_driver.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_xdna_driver.h @@ -184,6 +184,8 @@ public: uint32_t* size_copied, void* dest_mem_addr, bool* is_spm_data_loss) const override; + hsa_status_t IsModelEnabled(bool* enable) const override; + private: hsa_status_t QueryDriverVersion(); /// @brief Allocate device accesible heap space. diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/driver.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/driver.h index 52373bd829..0cf3dd71fe 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/driver.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/driver.h @@ -211,6 +211,10 @@ public: uint32_t* timeout, uint32_t* size_copied, void* dest_mem_addr, bool* is_spm_data_loss) const = 0; + /// @brief Check if the HSA KMT Model is enabled + /// @param[out] enable True if the model is enabled, false otherwise + virtual hsa_status_t IsModelEnabled(bool* enable) const = 0; + /// Unique identifier for supported kernel-mode drivers. const DriverType kernel_driver_type_; diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp index 0961cfd8f5..ced9a775a6 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp @@ -214,13 +214,21 @@ GpuAgent::GpuAgent(HSAuint32 node, const HsaNodeProperties& node_props, bool xna #if !defined(__linux__) wallclock_frequency_ = 0; #else - // Get wallclock freq from libdrm. - amdgpu_gpu_info info; - if (amdgpu_query_gpu_info(ldrm_dev_, &info) < 0) - throw AMD::hsa_exception(HSA_STATUS_ERROR, "Agent creation failed.\nlibdrm query failed.\n"); + bool model_enabled; + hsa_status_t status = driver().IsModelEnabled(&model_enabled); + assert(status == HSA_STATUS_SUCCESS && "IsModelEnabled failed"); + if (model_enabled) { + wallclock_frequency_ = 0; + } else { + // Get wallclock freq from libdrm. + amdgpu_gpu_info info; + if (amdgpu_query_gpu_info(ldrm_dev_, &info) < 0) + throw AMD::hsa_exception(HSA_STATUS_ERROR, "Agent creation failed.\nlibdrm query failed.\n"); + + // Reported by libdrm in KHz. + wallclock_frequency_ = uint64_t(info.gpu_counter_freq) * 1000ull; + } - // Reported by libdrm in KHz. - wallclock_frequency_ = uint64_t(info.gpu_counter_freq) * 1000ull; #endif auto& firstCpu = core::Runtime::runtime_singleton_->cpu_agents()[0]; diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_topology.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_topology.cpp index 10a310b485..c609e9e639 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_topology.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_topology.cpp @@ -425,6 +425,9 @@ bool Load() { if (core::Runtime::runtime_singleton_->AgentDrivers().empty()) return false; for (auto& d : core::Runtime::runtime_singleton_->AgentDrivers()) { + bool is_model_enabled = false; + d->IsModelEnabled(&is_model_enabled); + if (is_model_enabled) continue; if (!InitializeDriver(d)) return false; }