/* * Copyright © 2014 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation * files (the "Software"), to deal in the Software without * restriction, including without limitation the rights to use, copy, * modify, merge, publish, distribute, sublicense, and/or sell copies * of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including * the next paragraph) shall be included in all copies or substantial * portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */ #define _GNU_SOURCE #include #include #include #include #include #include #include #include #include #include #include #include "libhsakmt.h" #include "fmm.h" #define PAGE_SIZE 4096 #define MIN(X,Y) ((X) < (Y) ? (X) : (Y)) /* Number of memory banks added by thunk on top of topology */ #define NUM_OF_IGPU_HEAPS 3 #define NUM_OF_DGPU_HEAPS 3 /* SYSFS related */ #define KFD_SYSFS_PATH_GENERATION_ID "/sys/devices/virtual/kfd/kfd/topology/generation_id" #define KFD_SYSFS_PATH_SYSTEM_PROPERTIES "/sys/devices/virtual/kfd/kfd/topology/system_properties" #define KFD_SYSFS_PATH_NODES "/sys/devices/virtual/kfd/kfd/topology/nodes" #define PROC_CPUINFO_PATH "/proc/cpuinfo" typedef struct { uint32_t gpu_id; HsaNodeProperties node; HsaMemoryProperties *mem; /* node->NumBanks elements */ HsaCacheProperties *cache; HsaIoLinkProperties *link; } node_t; static HsaSystemProperties *_system = NULL; static node_t *node = NULL; static int processor_vendor; /* Supported System Vendors */ enum SUPPORTED_PROCESSOR_VENDORS { GENUINE_INTEL = 0, AUTHENTIC_AMD }; /* Adding newline to make the search easier */ static const char *supported_processor_vendor_name[] = { "GenuineIntel\n", "AuthenticAMD\n" }; static HSAKMT_STATUS topology_take_snapshot(void); static HSAKMT_STATUS topology_drop_snapshot(void); //static int get_cpu_stepping(uint16_t* stepping); static struct hsa_gfxip_table { uint16_t device_id; // Device ID unsigned char major; // GFXIP Major engine version unsigned char minor; // GFXIP Minor engine version unsigned char stepping; // GFXIP Stepping info unsigned char is_dgpu; // Predicate for dGPU devices const char* amd_name; // CALName of the device enum asic_family_type asic_family; } gfxip_lookup_table[] = { /* Kaveri Family */ { 0x1304, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, { 0x1305, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, { 0x1306, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, { 0x1307, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, { 0x1309, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, { 0x130A, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, { 0x130B, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, { 0x130C, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, { 0x130D, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, { 0x130E, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, { 0x130F, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, { 0x1310, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, { 0x1311, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, { 0x1312, 7, 0, 0, 0, "Spooky", CHIP_KAVERI }, { 0x1313, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, { 0x1315, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, { 0x1316, 7, 0, 0, 0, "Spooky", CHIP_KAVERI }, { 0x1317, 7, 0, 0, 0, "Spooky", CHIP_KAVERI }, { 0x1318, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, { 0x131B, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, { 0x131C, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, { 0x131D, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, /* Hawaii Family */ { 0x67A0, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, { 0x67A1, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, { 0x67A2, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, { 0x67A8, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, { 0x67A9, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, { 0x67AA, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, { 0x67B0, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, { 0x67B1, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, { 0x67B8, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, { 0x67B9, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, { 0x67BA, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, { 0x67BE, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, /* Carrizo Family */ { 0x9870, 8, 0, 1, 0, "Carrizo", CHIP_CARRIZO }, { 0x9874, 8, 0, 1, 0, "Carrizo", CHIP_CARRIZO }, { 0x9875, 8, 0, 1, 0, "Carrizo", CHIP_CARRIZO }, { 0x9876, 8, 0, 1, 0, "Carrizo", CHIP_CARRIZO }, { 0x9877, 8, 0, 1, 0, "Carrizo", CHIP_CARRIZO }, /* Tonga Family */ { 0x6920, 8, 0, 2, 1, "Tonga", CHIP_TONGA }, { 0x6921, 8, 0, 2, 1, "Tonga", CHIP_TONGA }, { 0x6928, 8, 0, 2, 1, "Tonga", CHIP_TONGA }, { 0x6929, 8, 0, 2, 1, "Tonga", CHIP_TONGA }, { 0x692B, 8, 0, 2, 1, "Tonga", CHIP_TONGA }, { 0x692F, 8, 0, 2, 1, "Tonga", CHIP_TONGA }, { 0x6930, 8, 0, 2, 1, "Tonga", CHIP_TONGA }, { 0x6938, 8, 0, 2, 1, "Tonga", CHIP_TONGA }, { 0x6939, 8, 0, 2, 1, "Tonga", CHIP_TONGA }, /* Fiji */ { 0x7300, 8, 0, 3, 1, "Fiji", CHIP_FIJI }, { 0x730F, 8, 0, 3, 1, "Fiji", CHIP_FIJI }, /* Polaris10 */ { 0x67C0, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, { 0x67C1, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, { 0x67C2, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, { 0x67C4, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, { 0x67C7, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, { 0x67C8, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, { 0x67C9, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, { 0x67CA, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, { 0x67CC, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, { 0x67CF, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, { 0x67D0, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, { 0x67DF, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, /* Polaris11 */ { 0x67E0, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, { 0x67E1, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, { 0x67E3, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, { 0x67E7, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, { 0x67E8, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, { 0x67E9, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, { 0x67EB, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, { 0x67EF, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, { 0x67FF, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, /* Vega10 */ { 0x6860, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, { 0x687F, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 } }; enum cache_type { CACHE_TYPE_NULL = 0, CACHE_TYPE_DATA = 1, CACHE_TYPE_INST = 2, CACHE_TYPE_UNIFIED = 3 }; typedef struct cacheinfo { HsaCacheProperties hsa_cache_prop; uint32_t num_threads_sharing; /* how many CPUs share this cache */ } cacheinfo_t; /* CPU cache table for all CPUs on the system. Each entry has the relative CPU * info and caches connected to that CPU. */ typedef struct cpu_cacheinfo { uint32_t len; /* length of the table -> number of online procs */ uint32_t num_caches; /* number of caches connected to this cpu */ uint32_t num_duplicated_caches; /* to count caches being shared */ uint32_t apicid; /* this cpu's apic id */ uint32_t max_num_apicid; /* max number of addressable IDs */ cacheinfo_t *cache_info; /* an array for cache information */ } cpu_cacheinfo_t; /* Deterministic Cache Parameters Leaf in cpuid */ union _cpuid_leaf_eax { /* Register EAX */ struct { enum cache_type type:5; uint32_t level:3; uint32_t is_self_initializing:1; uint32_t is_fully_associative:1; uint32_t reserved:4; uint32_t num_threads_sharing:12; uint32_t num_cores_on_die:6; } split; uint32_t full; }; union _cpuid_leaf_ebx { /* Register EBX */ struct { uint32_t coherency_line_size:12; uint32_t physical_line_partition:10; uint32_t ways_of_associativity:10; } split; uint32_t full; }; static void free_node(node_t *n) { assert(n); if (n == NULL) return; if ((n)->mem) free((n)->mem); if ((n)->cache) free((n)->cache); if ((n)->link) free((n)->link); } static void free_nodes(node_t *temp_nodes, int size) { int i; if (temp_nodes) { for (i = 0; i < size; i++) free_node(&temp_nodes[i]); free(temp_nodes); } } /* num_subdirs - find the number of sub-directories in the specified path * @dirpath - directory path to find sub-directories underneath * @prefix - only count sub-directory names starting with prefix. * Use blank string, "", to count all. * Return - number of sub-directories */ static int num_subdirs(char *dirpath, char *prefix) { int count = 0; DIR *dirp; struct dirent *dir; int prefix_len = strlen(prefix); dirp = opendir(dirpath); if(dirp) { while ((dir = readdir(dirp)) != 0) { if ((strcmp(dir->d_name, ".") == 0) || (strcmp(dir->d_name, "..") == 0)) continue; if (prefix_len && strncmp(dir->d_name, prefix, prefix_len)) continue; count++; } closedir(dirp); } return count; } #if defined(__x86_64__) || defined(__i386__) /* cpuid instruction returns processor identification and feature information * to the EAX, EBX, ECX, and EDX registers, as determined by input entered in * EAX (in some cases, ECX as well). */ static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, uint32_t *edx) { __asm__ __volatile__( "cpuid;" : "=a" (*eax), "=b" (*ebx), "=c" (*ecx), "=d" (*edx) : "0" (*eax), "2" (*ecx) : "memory" ); } /* In cases ECX is also used as an input for cpuid, i.e. cache leaf */ static void cpuid_count(uint32_t op, int count, uint32_t *eax, uint32_t *ebx, uint32_t *ecx, uint32_t *edx) { *eax = op; *ecx = count; cpuid(eax, ebx, ecx, edx); } /* Lock current process to the specified processor */ static int lock_to_processor(int processor) { cpu_set_t cpuset; memset(&cpuset, 0, sizeof(cpu_set_t)); CPU_SET(processor, &cpuset); /* 0: this process */ return sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); } /* Get count's order of 2. In other words, 2^rtn_value = count * When count is not an order of 2, round it up to the closest. */ static int get_count_order(unsigned int count) { int bit; uint32_t num; for (bit = 31; bit >= 0; bit--) { num = 1 << bit; if (count >= num) break; } if (count & (count - 1)) ++bit; return bit; } /* cpuid_find_num_cache_leaves - Use cpuid instruction to find out how many * cache leaves the CPU has. * @op - cpuid opcode to get cache information * Return - the number of cache leaves */ static int cpuid_find_num_cache_leaves(uint32_t op) { union _cpuid_leaf_eax eax; union _cpuid_leaf_ebx ebx; unsigned int ecx; unsigned int edx; int idx = -1; do { ++idx; cpuid_count(op, idx, &eax.full, &ebx.full, &ecx, &edx); } while (eax.split.type != CACHE_TYPE_NULL); return idx; } /* cpuid_get_cpu_cache_info - Use cpuid instruction to get cache information * @op - cpuid opcode to get cache information * @cpu_ci - this parameter is an input and also an output. * [IN] cpu_ci->num_caches: the number of caches of this cpu * [OUT] cpu_ci->cache_info: to store cache info collected */ static void cpuid_get_cpu_cache_info(uint32_t op, cpu_cacheinfo_t *cpu_ci) { union _cpuid_leaf_eax eax; union _cpuid_leaf_ebx ebx; uint32_t ecx; uint32_t edx; uint32_t index; cacheinfo_t *this_leaf; for (index = 0; index < cpu_ci->num_caches; index++) { cpuid_count(op, index, &eax.full, &ebx.full, &ecx, &edx); this_leaf = cpu_ci->cache_info + index; this_leaf->hsa_cache_prop.ProcessorIdLow = cpu_ci->apicid; this_leaf->num_threads_sharing = eax.split.num_threads_sharing + 1; this_leaf->hsa_cache_prop.CacheLevel = eax.split.level; this_leaf->hsa_cache_prop.CacheType.ui32.CPU = 1; if (eax.split.type & CACHE_TYPE_DATA ) this_leaf->hsa_cache_prop.CacheType.ui32.Data = 1; if (eax.split.type & CACHE_TYPE_INST ) this_leaf->hsa_cache_prop.CacheType.ui32.Instruction = 1; this_leaf->hsa_cache_prop.CacheLineSize = ebx.split.coherency_line_size + 1; this_leaf->hsa_cache_prop.CacheAssociativity = ebx.split.ways_of_associativity + 1; this_leaf->hsa_cache_prop.CacheLinesPerTag = ebx.split.physical_line_partition + 1; this_leaf->hsa_cache_prop.CacheSize = (ecx + 1) * (ebx.split.coherency_line_size + 1) * (ebx.split.physical_line_partition + 1) * (ebx.split.ways_of_associativity + 1); } } /* find_cpu_cache_siblings - In the cache list, some caches may be listed more * than once if they are shared by multiple CPUs. Identify the cache's CPU * siblings, record it to SiblingMap[], then remove the duplicated cache by * changing the cache size to 0. */ static void find_cpu_cache_siblings(cpu_cacheinfo_t *cpu_ci_list) { cacheinfo_t *this_leaf, *leaf2; uint32_t n, j, idx_msb, apicid1, apicid2; cpu_cacheinfo_t *this_cpu, *cpu2; uint32_t index; for (n = 0; n < cpu_ci_list->len; n++) { this_cpu = cpu_ci_list + n; for (index = 0; index < this_cpu->num_caches; index++) { this_leaf = this_cpu->cache_info + index; /* CacheSize 0 means an invalid cache */ if (!this_leaf->hsa_cache_prop.CacheSize) continue; if (this_leaf->num_threads_sharing == 1) // no siblings continue; idx_msb = get_count_order(this_leaf->num_threads_sharing); for (j = n + 1; j < cpu_ci_list->len; j++) { cpu2 = cpu_ci_list + j; leaf2 = cpu2->cache_info + index; apicid1 = this_leaf->hsa_cache_prop.ProcessorIdLow; apicid2 = leaf2->hsa_cache_prop.ProcessorIdLow; if ((apicid2 >> idx_msb) != (apicid1 >> idx_msb)) continue; /* A sibling leaf is found. Cache properties * use ProcIdLow as offset to represent siblings * in SiblingMap, so keep the lower apicid and * delete the other by changing CacheSize to 0. */ if (apicid1 < apicid2) { this_leaf->hsa_cache_prop.SiblingMap[0] = 1; this_leaf->hsa_cache_prop.SiblingMap[apicid2 - apicid1] = 1; leaf2->hsa_cache_prop.CacheSize = 0; cpu2->num_duplicated_caches++; } else { leaf2->hsa_cache_prop.SiblingMap[0] = 1; leaf2->hsa_cache_prop.SiblingMap[apicid1 - apicid2] = 1; this_leaf->hsa_cache_prop.CacheSize = 0; this_cpu->num_duplicated_caches++; } } } } } #endif /* X86 platform */ static HSAKMT_STATUS topology_sysfs_get_generation(uint32_t *gen) { FILE *fd; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; assert(gen); fd = fopen(KFD_SYSFS_PATH_GENERATION_ID, "r"); if (!fd) return HSAKMT_STATUS_ERROR; if (fscanf(fd, "%ul", gen) != 1) { ret = HSAKMT_STATUS_ERROR; goto err; } err: fclose(fd); return ret; } HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props) { FILE *fd; char *read_buf, *p; char prop_name[256]; long long unsigned int prop_val; uint32_t prog; int read_size; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; assert(props); fd = fopen(KFD_SYSFS_PATH_SYSTEM_PROPERTIES, "r"); if (!fd) return HSAKMT_STATUS_ERROR; read_buf = malloc(PAGE_SIZE); if (!read_buf) { ret = HSAKMT_STATUS_NO_MEMORY; goto err1; } read_size = fread(read_buf, 1, PAGE_SIZE, fd); if (read_size <= 0) { ret = HSAKMT_STATUS_ERROR; goto err2; } /* Since we're using the buffer as a string, we make sure the string terminates */ if(read_size >= PAGE_SIZE) read_size = PAGE_SIZE-1; read_buf[read_size] = 0; /* * Read the system properties */ prog = 0; p = read_buf; while(sscanf(p+=prog, "%s %llu\n%n", prop_name, &prop_val, &prog) == 2) { if (strcmp(prop_name,"platform_oem") == 0) props->PlatformOem = (uint32_t)prop_val; else if (strcmp(prop_name,"platform_id") == 0) props->PlatformId = (uint32_t)prop_val; else if (strcmp(prop_name,"platform_rev") == 0) props->PlatformRev = (uint32_t)prop_val; } /* * Discover the number of nodes: * Assuming that inside nodes folder there are only folders * which represent the node numbers */ props->NumNodes = num_subdirs(KFD_SYSFS_PATH_NODES, ""); err2: free(read_buf); err1: fclose(fd); return ret; } HSAKMT_STATUS topology_sysfs_get_gpu_id(uint32_t node_id, uint32_t *gpu_id) { FILE *fd; char path[256]; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; assert(gpu_id); snprintf(path, 256, "%s/%d/gpu_id", KFD_SYSFS_PATH_NODES, node_id); fd = fopen(path, "r"); if (!fd) return HSAKMT_STATUS_ERROR; if (fscanf(fd, "%ul", gpu_id) != 1) { ret = HSAKMT_STATUS_ERROR; } fclose(fd); return ret; } static const struct hsa_gfxip_table* find_hsa_gfxip_device(uint16_t device_id) { uint32_t i, table_size; table_size = sizeof(gfxip_lookup_table)/sizeof(struct hsa_gfxip_table); for (i=0; iasic_family; return HSAKMT_STATUS_SUCCESS; } bool topology_is_dgpu(uint16_t device_id) { const struct hsa_gfxip_table* hsa_gfxip = find_hsa_gfxip_device(device_id); if (hsa_gfxip && hsa_gfxip->is_dgpu) { is_dgpu = true; return true; } return false; } static HSAKMT_STATUS topology_get_cpu_model_name(HsaNodeProperties *props, bool is_apu) { FILE *fd; char read_buf[256], cpu_model_name[HSA_PUBLIC_NAME_SIZE]; const char *p; uint32_t i, apic_id; if (!props) return HSAKMT_STATUS_INVALID_PARAMETER; fd = fopen(PROC_CPUINFO_PATH, "r"); if (!fd) { printf("Failed to open [%s]. Unable to get CPU Model Name", PROC_CPUINFO_PATH); return HSAKMT_STATUS_ERROR; } while (fgets(read_buf, sizeof(read_buf), fd) != NULL) { /* Get the model name first, in case matching * apic IDs are also present in the file */ if (!strncmp("model name", read_buf, sizeof("model name") - 1)) { p = strrchr(read_buf, ':'); if (!p) goto err; p++; // remove separator ':' for (; isspace(*p); p++); /* remove white space */ /* Extract model name from string */ for (i = 0; i < sizeof(cpu_model_name) - 1 && p[i] != '\n'; i++) cpu_model_name[i] = p[i]; cpu_model_name[i] = '\0'; } if (!strncmp("apicid", read_buf, sizeof("apicid") - 1)) { p = strrchr(read_buf, ':'); if (!p) goto err; p++; // remove separator ':' for (; isspace(*p); p++); /* remove white space */ /* Extract apic_id from remaining chars */ apic_id = atoi(p); /* Set CPU model name only if corresponding apic id */ if (props->CComputeIdLo == apic_id) { /* Retrieve the CAL name of CPU node */ if (!is_apu) strncpy( (char *) props->AMDName, cpu_model_name, sizeof(props->AMDName)); /* Convert from UTF8 to UTF16 */ for (i = 0; cpu_model_name[i] != '\0' && i < HSA_PUBLIC_NAME_SIZE - 1; i++) props->MarketingName[i] = cpu_model_name[i]; props->MarketingName[i] = '\0'; } } } fclose(fd); return HSAKMT_STATUS_SUCCESS; err: fclose(fd); return HSAKMT_STATUS_ERROR; } static int topology_search_processor_vendor(const char *processor_name) { unsigned int i; for (i = 0; i < ARRAY_LEN(supported_processor_vendor_name); i++) { if (!strcmp(processor_name, supported_processor_vendor_name[i])) return i; } return -1; } /* topology_set_processor_vendor - Parse /proc/cpuinfo and * to find processor vendor and set global variable processor_vendor * * cat /proc/cpuinfo format is - "token : Value" * where token = "vendor_id" and * Value = indicates System Vendor */ static void topology_set_processor_vendor(void) { FILE *fd; char read_buf[256]; const char *p; fd = fopen(PROC_CPUINFO_PATH, "r"); if (!fd) { printf("Failed to open [%s]. Setting Processor Vendor to %s", PROC_CPUINFO_PATH, supported_processor_vendor_name[GENUINE_INTEL]); processor_vendor = GENUINE_INTEL; return; } while (fgets(read_buf, sizeof(read_buf), fd) != NULL) { if (!strncmp("vendor_id", read_buf, sizeof("vendor_id") - 1)) { p = strrchr(read_buf, ':'); p++; // remove separor ':' for (; isspace(*p); p++); /* remove white space */ processor_vendor = topology_search_processor_vendor(p); if (processor_vendor != -1) { fclose(fd); return; } } } fclose(fd); printf("Failed to get Processor Vendor. Setting to %s", supported_processor_vendor_name[GENUINE_INTEL]); processor_vendor = GENUINE_INTEL; } HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32_t *gpu_id, struct pci_access* pacc ) { FILE *fd; char *read_buf, *p; char prop_name[256]; char path[256]; long long unsigned int prop_val; uint32_t i, prog; uint16_t fw_version = 0; int read_size; const struct hsa_gfxip_table* hsa_gfxip; char namebuf[HSA_PUBLIC_NAME_SIZE]; const char* name; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; assert(props); assert(gpu_id); /* Retrieve the GPU ID */ ret = topology_sysfs_get_gpu_id(node_id, gpu_id); read_buf = malloc(PAGE_SIZE); if (!read_buf) return HSAKMT_STATUS_NO_MEMORY; /* Retrieve the node properties */ snprintf(path, 256, "%s/%d/properties", KFD_SYSFS_PATH_NODES, node_id); fd = fopen(path, "r"); if (!fd) { free(read_buf); return HSAKMT_STATUS_ERROR; } read_size = fread(read_buf, 1, PAGE_SIZE, fd); if (read_size <= 0) { ret = HSAKMT_STATUS_ERROR; goto err; } /* Since we're using the buffer as a string, we make sure the string terminates */ if(read_size >= PAGE_SIZE) read_size = PAGE_SIZE-1; read_buf[read_size] = 0; /* * Read the node properties */ prog = 0; p = read_buf; while(sscanf(p+=prog, "%s %llu\n%n", prop_name, &prop_val, &prog) == 2) { if (strcmp(prop_name,"cpu_cores_count") == 0) props->NumCPUCores = (uint32_t)prop_val; else if (strcmp(prop_name,"simd_count") == 0) props->NumFComputeCores = (uint32_t)prop_val; else if (strcmp(prop_name,"mem_banks_count") == 0) props->NumMemoryBanks = (uint32_t)prop_val; else if (strcmp(prop_name,"caches_count") == 0) props->NumCaches = (uint32_t)prop_val; else if (strcmp(prop_name,"io_links_count") == 0) props->NumIOLinks = (uint32_t)prop_val; else if (strcmp(prop_name,"cpu_core_id_base") == 0) props->CComputeIdLo = (uint32_t)prop_val; else if (strcmp(prop_name,"simd_id_base") == 0) props->FComputeIdLo = (uint32_t)prop_val; else if (strcmp(prop_name,"capability") == 0) props->Capability.Value = (uint32_t)prop_val; else if (strcmp(prop_name,"max_waves_per_simd") == 0) props->MaxWavesPerSIMD = (uint32_t)prop_val; else if (strcmp(prop_name,"lds_size_in_kb") == 0) props->LDSSizeInKB = (uint32_t)prop_val; else if (strcmp(prop_name,"gds_size_in_kb") == 0) props->GDSSizeInKB = (uint32_t)prop_val; else if (strcmp(prop_name,"wave_front_size") == 0) props->WaveFrontSize = (uint32_t)prop_val; else if (strcmp(prop_name,"array_count") == 0) props->NumShaderBanks = (uint32_t)prop_val; else if (strcmp(prop_name,"simd_arrays_per_engine") == 0) props->NumArrays = (uint32_t)prop_val; else if (strcmp(prop_name,"cu_per_simd_array") == 0) props->NumCUPerArray = (uint32_t)prop_val; else if (strcmp(prop_name,"simd_per_cu") == 0) props->NumSIMDPerCU = (uint32_t)prop_val; else if (strcmp(prop_name,"max_slots_scratch_cu") == 0) props->MaxSlotsScratchCU = (uint32_t)prop_val; else if (strcmp(prop_name,"fw_version") == 0) fw_version = (uint16_t)prop_val; else if (strcmp(prop_name,"vendor_id") == 0) props->VendorId = (uint32_t)prop_val; else if (strcmp(prop_name,"device_id") == 0) props->DeviceId = (uint32_t)prop_val; else if (strcmp(prop_name,"location_id") == 0) props->LocationId = (uint32_t)prop_val; else if (strcmp(prop_name,"max_engine_clk_fcompute") == 0) props->MaxEngineClockMhzFCompute = (uint32_t)prop_val; else if (strcmp(prop_name,"max_engine_clk_ccompute") == 0) props->MaxEngineClockMhzCCompute = (uint32_t)prop_val; else if (strcmp(prop_name,"local_mem_size") == 0) props->LocalMemSize = prop_val; } // get_cpu_stepping(&stepping); props->EngineId.ui32.uCode = fw_version & 0x3ff; props->EngineId.ui32.Major = 0; props->EngineId.ui32.Minor = 0; props->EngineId.ui32.Stepping = 0; hsa_gfxip = find_hsa_gfxip_device(props->DeviceId); if (hsa_gfxip) { props->EngineId.ui32.Major = hsa_gfxip->major & 0x3f; props->EngineId.ui32.Minor = hsa_gfxip->minor; props->EngineId.ui32.Stepping = hsa_gfxip->stepping; if (!hsa_gfxip->amd_name) { ret = HSAKMT_STATUS_ERROR; goto err; } /* Retrieve the CAL name of the node */ strncpy( (char *) props->AMDName, hsa_gfxip->amd_name, sizeof(props->AMDName) ); if (props->NumCPUCores) { /* Is APU node */ ret = topology_get_cpu_model_name(props, true); if (ret != HSAKMT_STATUS_SUCCESS) { printf("Failed to get APU Model Name from %s\n", PROC_CPUINFO_PATH); ret = HSAKMT_STATUS_SUCCESS; /* No hard error, continue regardless */ } } else { /* Is dGPU Node * Retrieve the marketing name of the node using pcilib, * convert UTF8 to UTF16 */ name = pci_lookup_name(pacc, namebuf, sizeof(namebuf), PCI_LOOKUP_DEVICE, props->VendorId, props->DeviceId); for (i = 0; name[i] != 0 && i < HSA_PUBLIC_NAME_SIZE - 1; i++) props->MarketingName[i] = name[i]; props->MarketingName[i] = '\0'; } } else { /* Is CPU Node */ if (!props->NumFComputeCores || !props->DeviceId) { ret = topology_get_cpu_model_name(props, false); if (ret != HSAKMT_STATUS_SUCCESS) { printf("Failed to get CPU Model Name from %s\n", PROC_CPUINFO_PATH); ret = HSAKMT_STATUS_SUCCESS; /* No hard error, continue regardless */ } } else { ret = HSAKMT_STATUS_ERROR; goto err; } } if (props->NumFComputeCores) assert(props->EngineId.ui32.Major); err: free(read_buf); fclose(fd); return ret; } static HSAKMT_STATUS topology_sysfs_get_mem_props(uint32_t node_id, uint32_t mem_id, HsaMemoryProperties *props) { FILE *fd; char *read_buf, *p; char prop_name[256]; char path[256]; long long unsigned int prop_val; uint32_t prog; int read_size; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; assert(props); snprintf(path, 256, "%s/%d/mem_banks/%d/properties", KFD_SYSFS_PATH_NODES, node_id, mem_id); fd = fopen(path, "r"); if (!fd) { return HSAKMT_STATUS_ERROR; } read_buf = malloc(PAGE_SIZE); if (!read_buf) { ret = HSAKMT_STATUS_NO_MEMORY; goto err1; } read_size = fread(read_buf, 1, PAGE_SIZE, fd); if (read_size <= 0) { ret = HSAKMT_STATUS_ERROR; goto err2; } /* Since we're using the buffer as a string, we make sure the string terminates */ if(read_size >= PAGE_SIZE) read_size = PAGE_SIZE-1; read_buf[read_size] = 0; prog = 0; p = read_buf; while(sscanf(p+=prog, "%s %llu\n%n", prop_name, &prop_val, &prog) == 2) { if (strcmp(prop_name,"heap_type") == 0) props->HeapType = (uint32_t)prop_val; else if (strcmp(prop_name,"size_in_bytes") == 0) props->SizeInBytes = (uint64_t)prop_val; else if (strcmp(prop_name,"flags") == 0) props->Flags.MemoryProperty = (uint32_t)prop_val; else if (strcmp(prop_name,"width") == 0) props->Width = (uint32_t)prop_val; else if (strcmp(prop_name,"mem_clk_max") == 0) props->MemoryClockMax = (uint32_t)prop_val; } err2: free(read_buf); err1: fclose(fd); return ret; } #if defined(__x86_64__) || defined(__i386__) /* topology_destroy_temp_cpu_cache_list - Free the memory allocated in * topology_create_temp_cpu_cache_list(). */ static void topology_destroy_temp_cpu_cache_list(void *temp_cpu_ci_list) { uint32_t n; cpu_cacheinfo_t *p_temp_cpu_ci_list = (cpu_cacheinfo_t *)temp_cpu_ci_list; cpu_cacheinfo_t *this_cpu; if (p_temp_cpu_ci_list) { for (n = 0; n < p_temp_cpu_ci_list->len; n++) { this_cpu = p_temp_cpu_ci_list + n; if (this_cpu->cache_info) free(this_cpu->cache_info); } free(p_temp_cpu_ci_list); } p_temp_cpu_ci_list = NULL; } /* topology_create_temp_cpu_cache_list - Create a temporary cpu-cache list to * store cpu cache information. This list will be used to copy * cache information to each CPU node. Must call * topology_destroy_temp_cpu_cache_list to free the memory after * the information is copied. * @temp_cpu_ci_list - [OUT] temporary cpu-cache-info list to store data * Return - HSAKMT_STATUS_SUCCESS in success or error number in failure */ static HSAKMT_STATUS topology_create_temp_cpu_cache_list(void **temp_cpu_ci_list) { HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; void *p_temp_cpu_ci_list; int procs_online; cpu_set_t orig_cpuset; int i; uint32_t cpuid_op_cache; uint32_t eax, ebx, ecx = 0, edx; /* cpuid registers */ cpu_cacheinfo_t *cpu_ci_list, *this_cpu; if (!temp_cpu_ci_list) { ret = HSAKMT_STATUS_ERROR; goto exit; } *temp_cpu_ci_list = NULL; procs_online = (int)sysconf(_SC_NPROCESSORS_ONLN); if (procs_online <= 0) { ret = HSAKMT_STATUS_ERROR; goto exit; } p_temp_cpu_ci_list = calloc(sizeof(cpu_cacheinfo_t) * procs_online, 1); if (!p_temp_cpu_ci_list) { ret = HSAKMT_STATUS_NO_MEMORY; goto exit; } cpu_ci_list = (cpu_cacheinfo_t *)p_temp_cpu_ci_list; cpu_ci_list->len = procs_online; if (processor_vendor == AUTHENTIC_AMD) cpuid_op_cache = 0x8000001d; else cpuid_op_cache = 0x4; /* lock_to_processor() changes the affinity. Save the current affinity * so we can restore it after cpuid is done. */ CPU_ZERO(&orig_cpuset); if (sched_getaffinity(0, sizeof(cpu_set_t), &orig_cpuset) != 0) { printf("Failed to get CPU affinity\n"); free(p_temp_cpu_ci_list); ret = HSAKMT_STATUS_ERROR; goto exit; } for (i = 0; i < procs_online; i++) { this_cpu = cpu_ci_list + i; lock_to_processor(i); /* so cpuid is executed in correct cpu */ eax = 0x1; cpuid(&eax, &ebx, &ecx, &edx); this_cpu->apicid = (ebx >> 24) & 0xff; this_cpu->max_num_apicid = (ebx >> 16) & 0x0FF; this_cpu->num_caches = cpuid_find_num_cache_leaves(cpuid_op_cache); this_cpu->num_duplicated_caches = 0; this_cpu->cache_info = calloc( sizeof(cacheinfo_t) * this_cpu->num_caches, 1); if (!this_cpu->cache_info) { ret = HSAKMT_STATUS_NO_MEMORY; goto err; } cpuid_get_cpu_cache_info(cpuid_op_cache, this_cpu); } find_cpu_cache_siblings(cpu_ci_list); *temp_cpu_ci_list = p_temp_cpu_ci_list; err: /* restore affinity to original */ sched_setaffinity(0, sizeof(cpu_set_t), &orig_cpuset); exit: if (ret != HSAKMT_STATUS_SUCCESS) topology_destroy_temp_cpu_cache_list(*temp_cpu_ci_list); return ret; } /* topology_get_cpu_cache_props - Read CPU cache information from the temporary * cache list and put them to the node's cache properties entry. * @tbl - the node table to fill up * @cpu_ci_list - the cpu cache information list to look up cache info * Return - HSAKMT_STATUS_SUCCESS in success or error number in failure */ static HSAKMT_STATUS topology_get_cpu_cache_props(node_t *tbl, cpu_cacheinfo_t *cpu_ci_list) { HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; uint32_t apicid_low = tbl->node.CComputeIdLo, apicid_max = 0; uint32_t n, cache_cnt, idx; cpu_cacheinfo_t *this_cpu; cacheinfo_t *this_leaf; /* CPU cache info list contains all CPUs. Find out CPUs belonging to * this node and number of caches under, so we can allocate the cache * properties in the node. */ tbl->node.NumCaches = 0; for (n = 0; n < cpu_ci_list->len; n++) { this_cpu = cpu_ci_list + n; if (this_cpu->apicid == apicid_low) /* found the first cpu in the node */ apicid_max = apicid_low + this_cpu->max_num_apicid - 1; if ((this_cpu->apicid < apicid_low) || (this_cpu->apicid > apicid_max)) continue; /* this cpu doesn't belong to the node */ tbl->node.NumCaches += this_cpu->num_caches - this_cpu->num_duplicated_caches; } tbl->cache = calloc( sizeof(HsaCacheProperties) * tbl->node.NumCaches, 1); if (!tbl->cache) { ret = HSAKMT_STATUS_NO_MEMORY; goto exit; } /* Now fill in the information to cache properties. */ cache_cnt = 0; for (n = 0; n < cpu_ci_list->len; n++) { this_cpu = cpu_ci_list + n; if ((this_cpu->apicid < apicid_low) || this_cpu->apicid > apicid_max) continue; /* this cpu doesn't belong to the node */ for (idx = 0; idx < this_cpu->num_caches; idx++) { this_leaf = this_cpu->cache_info + idx; if (this_leaf->hsa_cache_prop.CacheSize > 0) memcpy(&tbl->cache[cache_cnt++], &this_leaf->hsa_cache_prop, sizeof(HsaCacheProperties)); if (cache_cnt >= tbl->node.NumCaches) goto exit; } } exit: return ret; } #else /* not X86 */ static void topology_destroy_temp_cpu_cache_list(void *temp_cpu_ci_list) { } static HSAKMT_STATUS topology_create_temp_cpu_cache_list(void **temp_cpu_ci_list) { return HSAKMT_STATUS_SUCCESS; } static HSAKMT_STATUS topology_get_cpu_cache_props(node_t *tbl, cpu_cacheinfo_t *cpu_ci_list) { return HSAKMT_STATUS_SUCCESS; } #endif static HSAKMT_STATUS topology_sysfs_get_cache_props(uint32_t node_id, uint32_t cache_id, HsaCacheProperties *props) { FILE *fd; char *read_buf, *p; char prop_name[256]; char path[256]; long long unsigned int prop_val; uint32_t i, prog; int read_size; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; assert(props); snprintf(path, 256, "%s/%d/caches/%d/properties", KFD_SYSFS_PATH_NODES, node_id, cache_id); fd = fopen(path, "r"); if (!fd) { return HSAKMT_STATUS_ERROR; } read_buf = malloc(PAGE_SIZE); if (!read_buf) { ret = HSAKMT_STATUS_NO_MEMORY; goto err1; } read_size = fread(read_buf, 1, PAGE_SIZE, fd); if (read_size <= 0) { ret = HSAKMT_STATUS_ERROR; goto err2; } /* Since we're using the buffer as a string, we make sure the string terminates */ if(read_size >= PAGE_SIZE) read_size = PAGE_SIZE-1; read_buf[read_size] = 0; prog = 0; p = read_buf; while(sscanf(p+=prog, "%s %llu\n%n", prop_name, &prop_val, &prog) == 2) { if (strcmp(prop_name,"processor_id_low") == 0) props->ProcessorIdLow = (uint32_t)prop_val; else if (strcmp(prop_name,"level") == 0) props->CacheLevel = (uint32_t)prop_val; else if (strcmp(prop_name,"size") == 0) props->CacheSize = (uint32_t)prop_val; else if (strcmp(prop_name,"cache_line_size") == 0) props->CacheLineSize = (uint32_t)prop_val; else if (strcmp(prop_name,"cache_lines_per_tag") == 0) props->CacheLinesPerTag = (uint32_t)prop_val; else if (strcmp(prop_name,"association") == 0) props->CacheAssociativity = (uint32_t)prop_val; else if (strcmp(prop_name,"latency") == 0) props->CacheLatency = (uint32_t)prop_val; else if (strcmp(prop_name,"type") == 0) props->CacheType.Value = (uint32_t)prop_val; else if (strcmp(prop_name, "sibling_map") == 0) break; } prog = 0; if ((sscanf(p, "sibling_map %n", &prog)) == 0 && prog) { i = 0; while ((i < HSA_CPU_SIBLINGS) && (sscanf(p+=prog, "%u%*[,\n]%n", &props->SiblingMap[i++], &prog) == 1)); } err2: free(read_buf); err1: fclose(fd); return ret; } static HSAKMT_STATUS topology_sysfs_get_iolink_props(uint32_t node_id, uint32_t iolink_id, HsaIoLinkProperties *props) { FILE *fd; char *read_buf, *p; char prop_name[256]; char path[256]; long long unsigned int prop_val; uint32_t prog; int read_size; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; assert(props); snprintf(path, 256, "%s/%d/io_links/%d/properties", KFD_SYSFS_PATH_NODES, node_id, iolink_id); fd = fopen(path, "r"); if (!fd) { return HSAKMT_STATUS_ERROR; } read_buf = malloc(PAGE_SIZE); if (!read_buf) { ret = HSAKMT_STATUS_NO_MEMORY; goto err1; } read_size = fread(read_buf, 1, PAGE_SIZE, fd); if (read_size <= 0) { ret = HSAKMT_STATUS_ERROR; goto err2; } /* Since we're using the buffer as a string, we make sure the string terminates */ if(read_size >= PAGE_SIZE) read_size = PAGE_SIZE-1; read_buf[read_size] = 0; prog = 0; p = read_buf; while(sscanf(p+=prog, "%s %llu\n%n", prop_name, &prop_val, &prog) == 2) { if (strcmp(prop_name,"type") == 0) props->IoLinkType = (uint32_t)prop_val; else if (strcmp(prop_name,"version_major") == 0) props->VersionMajor = (uint32_t)prop_val; else if (strcmp(prop_name,"version_minor") == 0) props->VersionMinor = (uint32_t)prop_val; else if (strcmp(prop_name,"node_from") == 0) props->NodeFrom = (uint32_t)prop_val; else if (strcmp(prop_name,"node_to") == 0) props->NodeTo = (uint32_t)prop_val; else if (strcmp(prop_name,"weight") == 0) props->Weight = (uint32_t)prop_val; else if (strcmp(prop_name,"min_latency") == 0) props->MinimumLatency = (uint32_t)prop_val; else if (strcmp(prop_name,"max_latency") == 0) props->MaximumLatency = (uint32_t)prop_val; else if (strcmp(prop_name,"min_bandwidth") == 0) props->MinimumBandwidth = (uint32_t)prop_val; else if (strcmp(prop_name,"max_bandwidth") == 0) props->MaximumBandwidth = (uint32_t)prop_val; else if (strcmp(prop_name,"recommended_transfer_size") == 0) props->RecTransferSize = (uint32_t)prop_val; else if (strcmp(prop_name,"flags") == 0) props->Flags.LinkProperty = (uint32_t)prop_val; } err2: free(read_buf); err1: fclose(fd); return ret; } /* topology_get_numa_node_link_tye - Return NUMA node interconnect based * on processor vendor */ static HSA_IOLINKTYPE topology_get_numa_node_link_tye(void) { if (processor_vendor == GENUINE_INTEL) return HSA_IOLINK_TYPE_QPI_1_1; else if (processor_vendor == AUTHENTIC_AMD) return HSA_IOLINKTYPE_HYPERTRANSPORT; else return HSA_IOLINKTYPE_UNDEFINED; } /* topology_get_free_io_link_slot_for_node - For the given node_id, find the next * available free slot to add an io_link */ static HsaIoLinkProperties * topology_get_free_io_link_slot_for_node(uint32_t node_id, const HsaSystemProperties *sys_props, node_t *temp_nodes) { HsaIoLinkProperties *props; if (node_id >= sys_props->NumNodes) { printf("Invalid node [%d]\n", node_id); return NULL; } props = temp_nodes[node_id].link; if (!props) { printf("Error. No io_link reported for Node [%d]\n", node_id); return NULL; } if (temp_nodes[node_id].node.NumIOLinks >= sys_props->NumNodes - 1) { printf("Error. No more space for io_link for Node [%d]\n", node_id); return NULL; } return &props[temp_nodes[node_id].node.NumIOLinks]; } /* topology_add_io_link_for_node - If a free slot is available, * add io_link for the given Node. * TODO: Add other members of HsaIoLinkProperties */ static HSAKMT_STATUS topology_add_io_link_for_node(uint32_t node_id, const HsaSystemProperties *sys_props, node_t *temp_nodes, HSA_IOLINKTYPE IoLinkType, uint32_t NodeTo, uint32_t Weight) { HsaIoLinkProperties *props; props = topology_get_free_io_link_slot_for_node(node_id, sys_props, temp_nodes); if (!props) return HSAKMT_STATUS_NO_MEMORY; props->IoLinkType = IoLinkType; props->NodeFrom = node_id; props->NodeTo = NodeTo; props->Weight = Weight; temp_nodes[node_id].node.NumIOLinks++; return HSAKMT_STATUS_SUCCESS; } /* topology_create_qpi_links - Create QPI or HT links among all NUMA nodes * For now, assume all the nodes are interconnected with same Weight (=1) */ static void topology_create_qpi_links(const HsaSystemProperties *sys_props, node_t *temp_nodes) { unsigned int i, j; HSAKMT_STATUS ret; /* Find all CPU Nodes and connect each other via HT or QPI io_link */ for (i = 0; i < sys_props->NumNodes - 1; i++) { for (j = i + 1; j < sys_props->NumNodes; j++) { if (temp_nodes[i].gpu_id == 0 && temp_nodes[j].gpu_id == 0) { ret = topology_add_io_link_for_node(i, sys_props, temp_nodes, topology_get_numa_node_link_tye(), j, 1); if (ret != HSAKMT_STATUS_SUCCESS) printf("Error [%d]. Failed to add QPI link from Node [%d]->[%d]\n", ret, i, j); ret = topology_add_io_link_for_node(j, sys_props, temp_nodes, topology_get_numa_node_link_tye(), i, 1); if (ret != HSAKMT_STATUS_SUCCESS) printf("Error [%d]. Failed to add QPI link from Node [%d]->[%d]\n", ret, j, i); } } } } /* topology_create_reverse_io_link - Create io_links from the given CPU * NUMA node to all the GPUs attached to that node */ static void topology_create_reverse_io_link(uint32_t cpu_node, const HsaSystemProperties *sys_props, node_t *temp_nodes) { unsigned int gpu_node; HSAKMT_STATUS ret; for (gpu_node = 0; gpu_node < sys_props->NumNodes; gpu_node++) { if (temp_nodes[gpu_node].gpu_id != 0) { /* Check if this GPU is connected to the give cpu_node, * if so create an io_link */ if (temp_nodes[gpu_node].link->NodeTo == cpu_node) { ret = topology_add_io_link_for_node(cpu_node, sys_props, temp_nodes, HSA_IOLINKTYPE_PCIEXPRESS, gpu_node, temp_nodes[gpu_node].link->Weight); if (ret != HSAKMT_STATUS_SUCCESS) { printf("Error [%d]. Failed to create reverse io_links from Node [%d]\n", ret, cpu_node); return; } } } } } /* topology_create_indirect_gpu_links - For the given cpu_node, * find all nodes connected to it and create io_links * among them */ static void topology_create_indirect_gpu_links(uint32_t cpu_node, const HsaSystemProperties *sys_props, node_t *temp_nodes) { unsigned int i, j; HSAKMT_STATUS ret; HSA_IOLINKTYPE IoLinkType; HsaIoLinkProperties *props = temp_nodes[cpu_node].link; if (!props || temp_nodes[cpu_node].node.NumIOLinks == 0) { printf("CPU Node [%d] has no GPU connected\n", cpu_node); return; } /* props is the list of io_links cpu_node is connected to. * Make an indirect io_links from props[i].NodeTo --> props[j].NodeTo * and props[j].NodeTo --> props[i].NodeTo */ for (i = 0; i < temp_nodes[cpu_node].node.NumIOLinks - 1; i++) { for (j = i + 1; j < temp_nodes[cpu_node].node.NumIOLinks; j++) { /* Ignore CPU <--> CPU node connected as it is handled by QPI * link function */ if (temp_nodes[props[i].NodeTo].gpu_id == 0 && temp_nodes[props[j].NodeTo].gpu_id == 0) continue; /* For the given cpu_node, connect to or from the GPUs that are * connected directly to it via PCIEXPRESS */ if ((temp_nodes[props[i].NodeTo].gpu_id != 0 && props[i].IoLinkType != HSA_IOLINKTYPE_PCIEXPRESS) || (temp_nodes[props[j].NodeTo].gpu_id != 0 && props[j].IoLinkType != HSA_IOLINKTYPE_PCIEXPRESS)) continue; /* The link is from GPU to non-parent NUMA node. So set link type * to HT or QPI */ if (temp_nodes[props[i].NodeTo].gpu_id == 0 || temp_nodes[props[j].NodeTo].gpu_id == 0) IoLinkType = topology_get_numa_node_link_tye(); else IoLinkType = HSA_IOLINKTYPE_PCIEXPRESS; ret = topology_add_io_link_for_node(props[i].NodeTo, sys_props, temp_nodes, IoLinkType, props[j].NodeTo, props[i].Weight + props[j].Weight); if (ret != HSAKMT_STATUS_SUCCESS) printf("Error [%d]. Failed to add io_link from Node [%d]->[%d]\n", ret, i, j); ret = topology_add_io_link_for_node(props[j].NodeTo, sys_props, temp_nodes, IoLinkType, props[i].NodeTo, props[i].Weight + props[j].Weight); if (ret != HSAKMT_STATUS_SUCCESS) printf("Error [%d]. Failed to add io_link from Node [%d]->[%d]\n", ret, j, i); } } } HSAKMT_STATUS topology_take_snapshot(void) { uint32_t gen_start, gen_end, i, mem_id, cache_id, link_id; HsaSystemProperties sys_props; node_t *temp_nodes = 0; void *cpu_ci_list = NULL; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; struct pci_access *pacc; topology_set_processor_vendor(); retry: ret = topology_sysfs_get_generation(&gen_start); if (ret != HSAKMT_STATUS_SUCCESS) return ret; ret = topology_sysfs_get_system_props(&sys_props); if (ret != HSAKMT_STATUS_SUCCESS) return ret; if(sys_props.NumNodes > 0) { topology_create_temp_cpu_cache_list(&cpu_ci_list); temp_nodes = calloc(sys_props.NumNodes * sizeof(node_t),1); if (!temp_nodes) return HSAKMT_STATUS_NO_MEMORY; pacc = pci_alloc(); pci_init(pacc); for (i = 0; i < sys_props.NumNodes; i++) { ret = topology_sysfs_get_node_props(i, &temp_nodes[i].node, &temp_nodes[i].gpu_id, pacc); if (ret != HSAKMT_STATUS_SUCCESS) { free_nodes(temp_nodes, i); goto err; } if (temp_nodes[i].node.NumMemoryBanks) { temp_nodes[i].mem = calloc(temp_nodes[i].node.NumMemoryBanks * sizeof(HsaMemoryProperties), 1); if (!temp_nodes[i].mem) { ret = HSAKMT_STATUS_NO_MEMORY; free_nodes(temp_nodes, i + 1); goto err; } for (mem_id = 0; mem_id < temp_nodes[i].node.NumMemoryBanks; mem_id++) { ret = topology_sysfs_get_mem_props(i, mem_id, &temp_nodes[i].mem[mem_id]); if (ret != HSAKMT_STATUS_SUCCESS) { free_nodes(temp_nodes, i + 1); goto err; } } } if (temp_nodes[i].node.NumCaches) { temp_nodes[i].cache = calloc(temp_nodes[i].node.NumCaches * sizeof(HsaCacheProperties), 1); if (!temp_nodes[i].cache) { ret = HSAKMT_STATUS_NO_MEMORY; free_nodes(temp_nodes, i + 1); goto err; } for (cache_id = 0; cache_id < temp_nodes[i].node.NumCaches; cache_id++) { ret = topology_sysfs_get_cache_props(i, cache_id, &temp_nodes[i].cache[cache_id]); if (ret != HSAKMT_STATUS_SUCCESS) { free_nodes(temp_nodes, i + 1); goto err; } } } else if (!temp_nodes[i].gpu_id) { /* a CPU node */ ret = topology_get_cpu_cache_props( &temp_nodes[i], cpu_ci_list); if (ret != HSAKMT_STATUS_SUCCESS) { free_nodes(temp_nodes, i + 1); goto err; } } /* To simplify, allocate maximum needed memory for io_links for each node. This * removes the need for realloc when indirect and QPI links are added later */ temp_nodes[i].link = calloc(sys_props.NumNodes - 1, sizeof(HsaIoLinkProperties)); if (!temp_nodes[i].link) { ret = HSAKMT_STATUS_NO_MEMORY; free_nodes(temp_nodes, i + 1); goto err; } if (temp_nodes[i].node.NumIOLinks) { if (temp_nodes[i].gpu_id == 0) { printf("Warning. Not expecting CPU Node [%d] to have [%d] io_links.\n", i, temp_nodes[i].node.NumIOLinks); } for (link_id = 0; link_id < temp_nodes[i].node.NumIOLinks; link_id++) { ret = topology_sysfs_get_iolink_props(i, link_id, &temp_nodes[i].link[link_id]); if (ret != HSAKMT_STATUS_SUCCESS) { free_nodes(temp_nodes, i+1); goto err; } } } } pci_cleanup(pacc); } /* The Kernel only creates one way direct link - * GPU(PCI_BUS) --> Parent NUMA Node. Create the reverse direct * io_link here. [NUMA node] --> GPU */ /* Create the reverse io_link for all the CPU nodes */ for (i = 0; i < sys_props.NumNodes; i++) { if (temp_nodes[i].gpu_id == 0) { if (!temp_nodes[i].link) { printf("Unexpected NULL pointer. Node [%d].link\n", i); ret = HSAKMT_STATUS_NO_MEMORY; free_nodes(temp_nodes, i + 1); goto err; } topology_create_reverse_io_link(i, &sys_props, temp_nodes); } } /* Create QPI or HT links among CPU (NUMA) nodes. For now assume * all nodes are interconnected with same weight */ topology_create_qpi_links(&sys_props, temp_nodes); /* Create In-direct links for GPUs. Connect all the (Peer-to-Peer) GPUs * that belong to same NUMA node. * For each CPU (NUMA) node, interconnect all the GPUs. */ for (i = 0; i < sys_props.NumNodes; i++) { if (temp_nodes[i].gpu_id == 0) { topology_create_indirect_gpu_links(i, &sys_props, temp_nodes); } } ret = topology_sysfs_get_generation(&gen_end); if (ret != HSAKMT_STATUS_SUCCESS) { free_nodes(temp_nodes, sys_props.NumNodes); goto err; } if (gen_start != gen_end) { free_nodes(temp_nodes, sys_props.NumNodes); temp_nodes = 0; goto retry; } if (!_system) { _system = malloc(sizeof(HsaSystemProperties)); if (!_system) { free_nodes(temp_nodes, sys_props.NumNodes); return HSAKMT_STATUS_NO_MEMORY; } } *_system = sys_props; if (node) free(node); node = temp_nodes; err: topology_destroy_temp_cpu_cache_list(cpu_ci_list); return ret; } /* * Drop the Snashot of the HSA topology information. * Assume lock is held. */ HSAKMT_STATUS topology_drop_snapshot(void) { HSAKMT_STATUS err; if (!!_system != !!node) { printf("Probable inconsistency?\n"); err = HSAKMT_STATUS_SUCCESS; goto out; } if (node) { /* Remove state */ free_nodes(node, _system->NumNodes); node = NULL; } free(_system); _system = NULL; err = HSAKMT_STATUS_SUCCESS; out: return err; } HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id) { if (!node || !_system || _system->NumNodes <= nodeid) return HSAKMT_STATUS_INVALID_NODE_UNIT; if (gpu_id) *gpu_id = node[nodeid].gpu_id; return HSAKMT_STATUS_SUCCESS; } HSAKMT_STATUS gpuid_to_nodeid(uint32_t gpu_id, uint32_t* node_id){ uint64_t node_idx; for(node_idx = 0; node_idx < _system->NumNodes; node_idx++){ if (node[node_idx].gpu_id == gpu_id){ *node_id = node_idx; return HSAKMT_STATUS_SUCCESS; } } return HSAKMT_STATUS_INVALID_NODE_UNIT; } HSAKMT_STATUS HSAKMTAPI hsaKmtAcquireSystemProperties( HsaSystemProperties* SystemProperties //OUT ) { HSAKMT_STATUS err; CHECK_KFD_OPEN(); if (!SystemProperties) return HSAKMT_STATUS_INVALID_PARAMETER; pthread_mutex_lock(&hsakmt_mutex); err = topology_take_snapshot(); if (err != HSAKMT_STATUS_SUCCESS) goto out; assert(_system); *SystemProperties = *_system; err = HSAKMT_STATUS_SUCCESS; out: pthread_mutex_unlock(&hsakmt_mutex); return err; } HSAKMT_STATUS HSAKMTAPI hsaKmtReleaseSystemProperties(void) { CHECK_KFD_OPEN(); HSAKMT_STATUS err; pthread_mutex_lock(&hsakmt_mutex); err = topology_drop_snapshot(); pthread_mutex_unlock(&hsakmt_mutex); return err; } HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeProperties( HSAuint32 NodeId, //IN HsaNodeProperties* NodeProperties //OUT ) { HSAKMT_STATUS err; uint32_t gpu_id; if (!NodeProperties) return HSAKMT_STATUS_INVALID_PARAMETER; CHECK_KFD_OPEN(); pthread_mutex_lock(&hsakmt_mutex); /* KFD ADD page 18, snapshot protocol violation */ if (_system == NULL) { err = HSAKMT_STATUS_INVALID_NODE_UNIT; assert(_system); goto out; } if (NodeId >= _system->NumNodes) { err = HSAKMT_STATUS_INVALID_PARAMETER; goto out; } err = validate_nodeid(NodeId, &gpu_id); if (err != HSAKMT_STATUS_SUCCESS) return err; *NodeProperties = node[NodeId].node; /* For CPU only node don't add any additional GPU memory banks. */ if (gpu_id) { if (topology_is_dgpu(get_device_id_by_gpu_id(gpu_id))) NodeProperties->NumMemoryBanks += NUM_OF_DGPU_HEAPS; else NodeProperties->NumMemoryBanks += NUM_OF_IGPU_HEAPS; } err = HSAKMT_STATUS_SUCCESS; out: pthread_mutex_unlock(&hsakmt_mutex); return err; } HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeMemoryProperties( HSAuint32 NodeId, //IN HSAuint32 NumBanks, //IN HsaMemoryProperties* MemoryProperties //OUT ) { HSAKMT_STATUS err = HSAKMT_STATUS_SUCCESS; uint32_t i, gpu_id; HSAuint64 aperture_limit; bool nodeIsDGPU; if (!MemoryProperties) return HSAKMT_STATUS_INVALID_PARAMETER; CHECK_KFD_OPEN(); pthread_mutex_lock(&hsakmt_mutex); /* KFD ADD page 18, snapshot protocol violation */ if (_system == NULL) { err = HSAKMT_STATUS_INVALID_NODE_UNIT; assert(_system); goto out; } /* Check still necessary */ if (NodeId >= _system->NumNodes ) { err = HSAKMT_STATUS_INVALID_PARAMETER; goto out; } err = validate_nodeid(NodeId, &gpu_id); if (err != HSAKMT_STATUS_SUCCESS) goto out; memset(MemoryProperties, 0, NumBanks * sizeof(HsaMemoryProperties)); for (i = 0; i < MIN(node[NodeId].node.NumMemoryBanks, NumBanks); i++) { assert(node[NodeId].mem); MemoryProperties[i] = node[NodeId].mem[i]; } /* The following memory banks does not apply to CPU only node */ if (gpu_id == 0) goto out; nodeIsDGPU = topology_is_dgpu(get_device_id_by_gpu_id(gpu_id)); /*Add LDS*/ if (i < NumBanks && fmm_get_aperture_base_and_limit(FMM_LDS, gpu_id, &MemoryProperties[i].VirtualBaseAddress, &aperture_limit) == HSAKMT_STATUS_SUCCESS) { MemoryProperties[i].HeapType = HSA_HEAPTYPE_GPU_LDS; MemoryProperties[i].SizeInBytes = node[NodeId].node.LDSSizeInKB * 1024; i++; } /* Add Local memory - HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE. * For dGPU the topology node contains Local Memory and it is added by the for loop above */ if (!nodeIsDGPU && i < NumBanks && node[NodeId].node.LocalMemSize > 0 && fmm_get_aperture_base_and_limit(FMM_GPUVM, gpu_id, &MemoryProperties[i].VirtualBaseAddress, &aperture_limit) == HSAKMT_STATUS_SUCCESS) { MemoryProperties[i].HeapType = HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE; MemoryProperties[i].SizeInBytes = node[NodeId].node.LocalMemSize; i++; } /*Add SCRATCH*/ if (i < NumBanks && fmm_get_aperture_base_and_limit(FMM_SCRATCH, gpu_id, &MemoryProperties[i].VirtualBaseAddress, &aperture_limit) == HSAKMT_STATUS_SUCCESS) { MemoryProperties[i].HeapType = HSA_HEAPTYPE_GPU_SCRATCH; MemoryProperties[i].SizeInBytes = (aperture_limit - MemoryProperties[i].VirtualBaseAddress) + 1; i++; } /* On dGPUs add SVM aperture */ if (nodeIsDGPU && i < NumBanks && fmm_get_aperture_base_and_limit( FMM_SVM, gpu_id, &MemoryProperties[i].VirtualBaseAddress, &aperture_limit) == HSAKMT_STATUS_SUCCESS) { MemoryProperties[i].HeapType = HSA_HEAPTYPE_DEVICE_SVM; MemoryProperties[i].SizeInBytes = (aperture_limit - MemoryProperties[i].VirtualBaseAddress) + 1; i++; } out: pthread_mutex_unlock(&hsakmt_mutex); return err; } HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeCacheProperties( HSAuint32 NodeId, //IN HSAuint32 ProcessorId, //IN HSAuint32 NumCaches, //IN HsaCacheProperties* CacheProperties //OUT ) { HSAKMT_STATUS err; uint32_t i; if (!CacheProperties) return HSAKMT_STATUS_INVALID_PARAMETER; CHECK_KFD_OPEN(); pthread_mutex_lock(&hsakmt_mutex); /* KFD ADD page 18, snapshot protocol violation */ if (_system == NULL) { err = HSAKMT_STATUS_INVALID_NODE_UNIT; assert(_system); goto out; } if (NodeId >= _system->NumNodes || NumCaches > node[NodeId].node.NumCaches) { err = HSAKMT_STATUS_INVALID_PARAMETER; goto out; } for (i = 0; i < MIN(node[NodeId].node.NumCaches, NumCaches); i++) { assert(node[NodeId].cache); CacheProperties[i] = node[NodeId].cache[i]; } err = HSAKMT_STATUS_SUCCESS; out: pthread_mutex_unlock(&hsakmt_mutex); return err; } HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeIoLinkProperties( HSAuint32 NodeId, //IN HSAuint32 NumIoLinks, //IN HsaIoLinkProperties* IoLinkProperties //OUT ) { HSAKMT_STATUS err; uint32_t i; if (!IoLinkProperties) return HSAKMT_STATUS_INVALID_PARAMETER; CHECK_KFD_OPEN(); pthread_mutex_lock(&hsakmt_mutex); /* KFD ADD page 18, snapshot protocol violation */ if (_system == NULL) { err = HSAKMT_STATUS_INVALID_NODE_UNIT; assert(_system); goto out; } if (NodeId >= _system->NumNodes || NumIoLinks > node[NodeId].node.NumIOLinks) { err = HSAKMT_STATUS_INVALID_PARAMETER; goto out; } for (i = 0; i < MIN(node[NodeId].node.NumIOLinks, NumIoLinks); i++) { assert(node[NodeId].link); IoLinkProperties[i] = node[NodeId].link[i]; } err = HSAKMT_STATUS_SUCCESS; out: pthread_mutex_unlock(&hsakmt_mutex); return err; } uint16_t get_device_id_by_node(HSAuint32 node_id) { if (!node || !_system || _system->NumNodes <= node_id) return 0; return node[node_id].node.DeviceId; } uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id) { unsigned int i; if (!node || !_system) return 0; for (i = 0; i < _system->NumNodes; i++) { if (node[i].gpu_id == gpu_id) return node[i].node.DeviceId; } return 0; } HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array, uint32_t NumberOfNodes, uint32_t *NodeArray) { HSAKMT_STATUS ret; unsigned int i; if (NumberOfNodes == 0 || NodeArray == NULL || gpu_id_array == NULL) return HSAKMT_STATUS_INVALID_PARAMETER; /* Translate Node IDs to gpu_ids */ *gpu_id_array = malloc(NumberOfNodes * sizeof(uint32_t)); if (*gpu_id_array == NULL) return HSAKMT_STATUS_NO_MEMORY; for (i = 0; i < NumberOfNodes; i++) { ret = validate_nodeid(NodeArray[i], *gpu_id_array + i); if (ret != HSAKMT_STATUS_SUCCESS) { free(*gpu_id_array); break; } } return ret; } #if 0 static int get_cpu_stepping(uint16_t* stepping) { int ret; FILE* fd = fopen("/proc/cpuinfo", "r"); if (!fd) return -1; char* read_buf = malloc(PAGE_SIZE); if (!read_buf) { ret = -1; goto err1; } int read_size = fread(read_buf, 1, PAGE_SIZE, fd); if (read_size <= 0) { ret = -2; goto err2; } /* Since we're using the buffer as a string, we make sure the string terminates */ if(read_size >= PAGE_SIZE) read_size = PAGE_SIZE-1; read_buf[read_size] = 0; *stepping = 0; char* p = strstr(read_buf, "stepping"); if (p) sscanf(p , "stepping\t: %hu\n", stepping); err2: free(read_buf); err1: fclose(fd); return ret; } #endif